shamela 1.3.1 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +228 -391
- package/dist/index.d.ts +1 -23
- package/dist/index.js +5 -5
- package/dist/index.js.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -18,25 +18,21 @@ A universal TypeScript library for accessing and downloading Maktabah Shamela v4
|
|
|
18
18
|
## Table of Contents
|
|
19
19
|
|
|
20
20
|
- [Installation](#installation)
|
|
21
|
-
- [
|
|
22
|
-
- [
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
21
|
+
- [Quick Start](#quick-start)
|
|
22
|
+
- [Standard Node.js](#standard-nodejs)
|
|
23
|
+
- [Next.js / Bundled Environments](#nextjs--bundled-environments)
|
|
24
|
+
- [Browser](#browser)
|
|
25
|
+
- [API Reference](#api-reference)
|
|
26
|
+
- [getMasterMetadata](#getmastermetadata)
|
|
27
|
+
- [downloadMasterDatabase](#downloadmasterdatabase)
|
|
28
|
+
- [getBookMetadata](#getbookmetadata)
|
|
29
|
+
- [downloadBook](#downloadbook)
|
|
30
|
+
- [getBook](#getbook)
|
|
31
|
+
- [getMaster](#getmaster)
|
|
32
|
+
- [getCoverUrl](#getcoverurl)
|
|
33
33
|
- [Examples](#examples)
|
|
34
|
-
- [Downloading the Master Database](#downloading-the-master-database)
|
|
35
|
-
- [Downloading a Book](#downloading-a-book)
|
|
36
|
-
- [Retrieving Book Data](#retrieving-book-data)
|
|
37
|
-
- [Retrieving Master Data in memory](#retrieving-master-data-in-memory)
|
|
38
34
|
- [Data Structures](#data-structures)
|
|
39
|
-
- [Next.js
|
|
35
|
+
- [Next.js Demo](#nextjs-demo)
|
|
40
36
|
- [Troubleshooting](#troubleshooting)
|
|
41
37
|
- [Testing](#testing)
|
|
42
38
|
- [License](#license)
|
|
@@ -44,103 +40,53 @@ A universal TypeScript library for accessing and downloading Maktabah Shamela v4
|
|
|
44
40
|
## Installation
|
|
45
41
|
|
|
46
42
|
```bash
|
|
47
|
-
|
|
43
|
+
npm install shamela
|
|
48
44
|
```
|
|
49
45
|
|
|
50
|
-
or
|
|
51
|
-
|
|
52
46
|
```bash
|
|
53
|
-
|
|
47
|
+
bun add shamela
|
|
54
48
|
```
|
|
55
49
|
|
|
56
|
-
or
|
|
57
|
-
|
|
58
50
|
```bash
|
|
59
51
|
yarn add shamela
|
|
60
52
|
```
|
|
61
53
|
|
|
62
|
-
or
|
|
63
|
-
|
|
64
54
|
```bash
|
|
65
55
|
pnpm install shamela
|
|
66
56
|
```
|
|
67
57
|
|
|
68
|
-
##
|
|
58
|
+
## Quick Start
|
|
69
59
|
|
|
70
|
-
|
|
60
|
+
### Standard Node.js
|
|
71
61
|
|
|
72
|
-
|
|
73
|
-
- `SHAMELA_API_MASTER_PATCH_ENDPOINT`: The endpoint URL for the master database patches.
|
|
74
|
-
- `SHAMELA_API_BOOKS_ENDPOINT`: The base endpoint URL for book-related API calls.
|
|
75
|
-
- `SHAMELA_SQLJS_WASM_URL` (optional): Override the default CDN URL used to load the `sql.js` WebAssembly binary when running in the browser.
|
|
62
|
+
For simple Node.js scripts (non-bundled environments), the library auto-detects the WASM file:
|
|
76
63
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
```dotenv
|
|
80
|
-
SHAMELA_API_KEY=your_api_key_here
|
|
81
|
-
SHAMELA_API_MASTER_PATCH_ENDPOINT=https://shamela.ws/api/master_patch
|
|
82
|
-
SHAMELA_API_BOOKS_ENDPOINT=https://shamela.ws/api/books
|
|
83
|
-
# Optional when you host sql-wasm.wasm yourself
|
|
84
|
-
# SHAMELA_SQLJS_WASM_URL=https://example.com/sql-wasm.wasm
|
|
85
|
-
```
|
|
86
|
-
|
|
87
|
-
### Runtime configuration (browsers and serverless)
|
|
88
|
-
|
|
89
|
-
When you cannot rely on environment variables—such as when running inside a browser, an edge worker, or a serverless function—use the `configure` helper to provide credentials at runtime:
|
|
90
|
-
|
|
91
|
-
```ts
|
|
92
|
-
import { configure } from 'shamela';
|
|
64
|
+
```typescript
|
|
65
|
+
import { configure, getBook } from 'shamela';
|
|
93
66
|
|
|
67
|
+
// Configure API credentials
|
|
94
68
|
configure({
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
sqlJsWasmUrl: '/assets/sql-wasm.wasm',
|
|
100
|
-
// Optional: integrate with your application's logging system
|
|
101
|
-
logger: console,
|
|
102
|
-
// Optional: provide a custom fetch implementation (for tests or SSR)
|
|
103
|
-
fetchImplementation: fetch,
|
|
69
|
+
apiKey: process.env.SHAMELA_API_KEY,
|
|
70
|
+
booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,
|
|
71
|
+
masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,
|
|
72
|
+
// sqlJsWasmUrl is auto-detected in standard Node.js
|
|
104
73
|
});
|
|
105
|
-
```
|
|
106
|
-
|
|
107
|
-
You can call `configure` multiple times—values are merged, so later calls update only the keys you pass in.
|
|
108
|
-
|
|
109
|
-
The optional `logger` must expose `debug`, `info`, `warn`, and `error` methods. When omitted, the library stays silent by default.
|
|
110
|
-
|
|
111
|
-
## Usage
|
|
112
74
|
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
```javascript
|
|
118
|
-
import {
|
|
119
|
-
getMasterMetadata,
|
|
120
|
-
downloadMasterDatabase,
|
|
121
|
-
getBookMetadata,
|
|
122
|
-
downloadBook,
|
|
123
|
-
getBook,
|
|
124
|
-
getMaster,
|
|
125
|
-
getCoverUrl,
|
|
126
|
-
} from 'shamela';
|
|
75
|
+
// Use the library
|
|
76
|
+
const book = await getBook(26592);
|
|
77
|
+
console.log(`Downloaded book with ${book.pages.length} pages`);
|
|
127
78
|
```
|
|
128
79
|
|
|
129
80
|
### Next.js / Bundled Environments
|
|
130
81
|
|
|
131
|
-
|
|
82
|
+
For Next.js, webpack, Turbopack, and other bundlers, you need to explicitly configure the WASM file path.
|
|
132
83
|
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
Add the following to your `next.config.js` or `next.config.ts`:
|
|
84
|
+
**1. Update `next.config.ts` or `next.config.js`:**
|
|
136
85
|
|
|
137
86
|
```typescript
|
|
138
87
|
import type { NextConfig } from 'next';
|
|
139
88
|
|
|
140
89
|
const nextConfig: NextConfig = {
|
|
141
|
-
experimental: {
|
|
142
|
-
serverComponentsExternalPackages: ['shamela', 'sql.js'],
|
|
143
|
-
},
|
|
144
90
|
serverExternalPackages: ['shamela', 'sql.js'],
|
|
145
91
|
// ... rest of your config
|
|
146
92
|
};
|
|
@@ -148,30 +94,7 @@ const nextConfig: NextConfig = {
|
|
|
148
94
|
export default nextConfig;
|
|
149
95
|
```
|
|
150
96
|
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
#### 2. Create a server-only configuration file
|
|
154
|
-
|
|
155
|
-
Create a configuration file that will be imported only in server-side code:
|
|
156
|
-
|
|
157
|
-
**Option A: Using the `createNodeConfig` helper (Recommended)**
|
|
158
|
-
|
|
159
|
-
```typescript
|
|
160
|
-
// lib/shamela-server.ts
|
|
161
|
-
import { configure, createNodeConfig } from 'shamela';
|
|
162
|
-
|
|
163
|
-
// Configure once when this module loads
|
|
164
|
-
configure(createNodeConfig({
|
|
165
|
-
apiKey: process.env.SHAMELA_API_KEY,
|
|
166
|
-
booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,
|
|
167
|
-
masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,
|
|
168
|
-
}));
|
|
169
|
-
|
|
170
|
-
// Re-export the functions you need
|
|
171
|
-
export { getBookMetadata, downloadBook, getMaster, getBook } from 'shamela';
|
|
172
|
-
```
|
|
173
|
-
|
|
174
|
-
**Option B: Manual configuration**
|
|
97
|
+
**2. Create a server-only configuration file:**
|
|
175
98
|
|
|
176
99
|
```typescript
|
|
177
100
|
// lib/shamela-server.ts
|
|
@@ -185,10 +108,10 @@ configure({
|
|
|
185
108
|
masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,
|
|
186
109
|
});
|
|
187
110
|
|
|
188
|
-
export {
|
|
111
|
+
export { downloadBook, getBook, getBookMetadata, getMaster, downloadMasterDatabase } from 'shamela';
|
|
189
112
|
```
|
|
190
113
|
|
|
191
|
-
|
|
114
|
+
**3. Use in Server Actions:**
|
|
192
115
|
|
|
193
116
|
```typescript
|
|
194
117
|
'use server';
|
|
@@ -197,19 +120,35 @@ import { getBookMetadata, downloadBook } from '@/lib/shamela-server';
|
|
|
197
120
|
|
|
198
121
|
export async function downloadBookAction(bookId: number) {
|
|
199
122
|
const metadata = await getBookMetadata(bookId);
|
|
200
|
-
|
|
123
|
+
return await downloadBook(bookId, {
|
|
201
124
|
bookMetadata: metadata,
|
|
202
125
|
outputFile: { path: `./books/${bookId}.db` }
|
|
203
126
|
});
|
|
204
|
-
return result;
|
|
205
127
|
}
|
|
206
128
|
```
|
|
207
129
|
|
|
208
|
-
**Important:**
|
|
130
|
+
**Important:** Only import `shamela` in server-side code (Server Actions, API Routes, or Server Components). Never import in client components or `layout.tsx`.
|
|
131
|
+
|
|
132
|
+
### Browser
|
|
209
133
|
|
|
210
|
-
|
|
134
|
+
In browsers, the library automatically uses a CDN-hosted WASM file:
|
|
211
135
|
|
|
212
|
-
|
|
136
|
+
```typescript
|
|
137
|
+
import { configure, getBook } from 'shamela';
|
|
138
|
+
|
|
139
|
+
configure({
|
|
140
|
+
apiKey: 'your-api-key',
|
|
141
|
+
booksEndpoint: 'https://SHAMELA_INSTANCE.ws/api/books',
|
|
142
|
+
masterPatchEndpoint: 'https://SHAMELA_INSTANCE.ws/api/master_patch',
|
|
143
|
+
// Automatically uses CDN: https://cdn.jsdelivr.net/npm/sql.js@1.13.0/dist/sql-wasm.wasm
|
|
144
|
+
});
|
|
145
|
+
|
|
146
|
+
const book = await getBook(26592);
|
|
147
|
+
```
|
|
148
|
+
|
|
149
|
+
## API Reference
|
|
150
|
+
|
|
151
|
+
### getMasterMetadata
|
|
213
152
|
|
|
214
153
|
Fetches metadata for the master database.
|
|
215
154
|
|
|
@@ -217,57 +156,49 @@ Fetches metadata for the master database.
|
|
|
217
156
|
getMasterMetadata(version?: number): Promise<GetMasterMetadataResponsePayload>
|
|
218
157
|
```
|
|
219
158
|
|
|
220
|
-
- `version` (optional): The version number
|
|
159
|
+
- `version` (optional): The version number to check for updates (defaults to 0)
|
|
221
160
|
|
|
222
|
-
**Returns:** Promise
|
|
161
|
+
**Returns:** Promise resolving to master database metadata including download URL and version
|
|
223
162
|
|
|
224
163
|
**Example:**
|
|
225
164
|
|
|
226
|
-
```
|
|
227
|
-
const
|
|
228
|
-
console.log(
|
|
229
|
-
console.log(
|
|
165
|
+
```typescript
|
|
166
|
+
const metadata = await getMasterMetadata();
|
|
167
|
+
console.log(metadata.url); // Download URL
|
|
168
|
+
console.log(metadata.version); // Version number
|
|
230
169
|
|
|
231
170
|
// Check for updates from a specific version
|
|
232
171
|
const updates = await getMasterMetadata(5);
|
|
233
172
|
```
|
|
234
173
|
|
|
235
|
-
|
|
174
|
+
### downloadMasterDatabase
|
|
236
175
|
|
|
237
|
-
Downloads the master database
|
|
176
|
+
Downloads the master database containing all books, authors, and categories.
|
|
238
177
|
|
|
239
178
|
```typescript
|
|
240
179
|
downloadMasterDatabase(options: DownloadMasterOptions): Promise<string>
|
|
241
180
|
```
|
|
242
181
|
|
|
243
|
-
- `options
|
|
244
|
-
|
|
245
|
-
- `outputFile`: Object with `path` property specifying the output file path
|
|
182
|
+
- `options.masterMetadata` (optional): Pre-fetched metadata
|
|
183
|
+
- `options.outputFile.path`: Output file path (`.db`, `.sqlite`, or `.json`)
|
|
246
184
|
|
|
247
|
-
**Returns:** Promise
|
|
185
|
+
**Returns:** Promise resolving to the output file path
|
|
248
186
|
|
|
249
187
|
**Example:**
|
|
250
188
|
|
|
251
|
-
```
|
|
189
|
+
```typescript
|
|
252
190
|
// Download as SQLite database
|
|
253
191
|
await downloadMasterDatabase({
|
|
254
|
-
|
|
192
|
+
outputFile: { path: './master.db' }
|
|
255
193
|
});
|
|
256
194
|
|
|
257
195
|
// Download as JSON
|
|
258
196
|
await downloadMasterDatabase({
|
|
259
|
-
|
|
260
|
-
});
|
|
261
|
-
|
|
262
|
-
// Use pre-fetched metadata for efficiency
|
|
263
|
-
const masterMetadata = await getMasterMetadata();
|
|
264
|
-
await downloadMasterDatabase({
|
|
265
|
-
masterMetadata,
|
|
266
|
-
outputFile: { path: './master.db' },
|
|
197
|
+
outputFile: { path: './master.json' }
|
|
267
198
|
});
|
|
268
199
|
```
|
|
269
200
|
|
|
270
|
-
|
|
201
|
+
### getBookMetadata
|
|
271
202
|
|
|
272
203
|
Fetches metadata for a specific book.
|
|
273
204
|
|
|
@@ -275,105 +206,90 @@ Fetches metadata for a specific book.
|
|
|
275
206
|
getBookMetadata(id: number, options?: GetBookMetadataOptions): Promise<GetBookMetadataResponsePayload>
|
|
276
207
|
```
|
|
277
208
|
|
|
278
|
-
- `id`:
|
|
279
|
-
- `options` (optional):
|
|
280
|
-
|
|
281
|
-
- `minorVersion`: The minor version to check against
|
|
209
|
+
- `id`: Book identifier
|
|
210
|
+
- `options.majorVersion` (optional): Major version to check
|
|
211
|
+
- `options.minorVersion` (optional): Minor version to check
|
|
282
212
|
|
|
283
|
-
**Returns:** Promise
|
|
213
|
+
**Returns:** Promise resolving to book metadata
|
|
284
214
|
|
|
285
215
|
**Example:**
|
|
286
216
|
|
|
287
|
-
```
|
|
217
|
+
```typescript
|
|
288
218
|
const metadata = await getBookMetadata(26592);
|
|
289
|
-
console.log(metadata.majorReleaseUrl);
|
|
290
|
-
console.log(metadata.
|
|
291
|
-
|
|
292
|
-
// Check specific versions
|
|
293
|
-
const versionedMetadata = await getBookMetadata(26592, {
|
|
294
|
-
majorVersion: 1,
|
|
295
|
-
minorVersion: 2,
|
|
296
|
-
});
|
|
219
|
+
console.log(metadata.majorReleaseUrl);
|
|
220
|
+
console.log(metadata.minorReleaseUrl);
|
|
297
221
|
```
|
|
298
222
|
|
|
299
|
-
|
|
223
|
+
### downloadBook
|
|
300
224
|
|
|
301
|
-
Downloads and processes a book from
|
|
225
|
+
Downloads and processes a book from Shamela.
|
|
302
226
|
|
|
303
227
|
```typescript
|
|
304
228
|
downloadBook(id: number, options: DownloadBookOptions): Promise<string>
|
|
305
229
|
```
|
|
306
230
|
|
|
307
|
-
- `id`:
|
|
308
|
-
- `options
|
|
309
|
-
|
|
310
|
-
- `outputFile`: Object with `path` property specifying the output file path
|
|
231
|
+
- `id`: Book identifier
|
|
232
|
+
- `options.bookMetadata` (optional): Pre-fetched metadata
|
|
233
|
+
- `options.outputFile.path`: Output file path (`.db`, `.sqlite`, or `.json`)
|
|
311
234
|
|
|
312
|
-
**Returns:** Promise
|
|
235
|
+
**Returns:** Promise resolving to the output file path
|
|
313
236
|
|
|
314
237
|
**Example:**
|
|
315
238
|
|
|
316
|
-
```
|
|
239
|
+
```typescript
|
|
317
240
|
// Download as JSON
|
|
318
241
|
await downloadBook(26592, {
|
|
319
|
-
|
|
320
|
-
});
|
|
321
|
-
|
|
322
|
-
// Download as SQLite database
|
|
323
|
-
await downloadBook(26592, {
|
|
324
|
-
outputFile: { path: './book.db' },
|
|
242
|
+
outputFile: { path: './book.json' }
|
|
325
243
|
});
|
|
326
244
|
|
|
327
|
-
//
|
|
328
|
-
const bookMetadata = await getBookMetadata(26592);
|
|
245
|
+
// Download as SQLite
|
|
329
246
|
await downloadBook(26592, {
|
|
330
|
-
|
|
331
|
-
outputFile: { path: './book.db' },
|
|
247
|
+
outputFile: { path: './book.db' }
|
|
332
248
|
});
|
|
333
249
|
```
|
|
334
250
|
|
|
335
|
-
|
|
251
|
+
### getBook
|
|
336
252
|
|
|
337
|
-
Retrieves complete book data as a JavaScript object.
|
|
253
|
+
Retrieves complete book data as a JavaScript object.
|
|
338
254
|
|
|
339
255
|
```typescript
|
|
340
256
|
getBook(id: number): Promise<BookData>
|
|
341
257
|
```
|
|
342
258
|
|
|
343
|
-
- `id`:
|
|
259
|
+
- `id`: Book identifier
|
|
344
260
|
|
|
345
|
-
**Returns:** Promise
|
|
261
|
+
**Returns:** Promise resolving to book data with pages and titles
|
|
346
262
|
|
|
347
263
|
**Example:**
|
|
348
264
|
|
|
349
|
-
```
|
|
350
|
-
const
|
|
351
|
-
console.log(
|
|
352
|
-
console.log(
|
|
353
|
-
console.log(
|
|
265
|
+
```typescript
|
|
266
|
+
const book = await getBook(26592);
|
|
267
|
+
console.log(book.pages.length);
|
|
268
|
+
console.log(book.titles?.length);
|
|
269
|
+
console.log(book.pages[0].content);
|
|
354
270
|
```
|
|
355
271
|
|
|
356
|
-
|
|
272
|
+
### getMaster
|
|
357
273
|
|
|
358
|
-
Retrieves the entire master dataset
|
|
359
|
-
API reports for the snapshot.
|
|
274
|
+
Retrieves the entire master dataset as a JavaScript object.
|
|
360
275
|
|
|
361
276
|
```typescript
|
|
362
277
|
getMaster(): Promise<MasterData>
|
|
363
278
|
```
|
|
364
279
|
|
|
365
|
-
**Returns:** Promise
|
|
280
|
+
**Returns:** Promise resolving to master data with authors, books, categories, and version
|
|
366
281
|
|
|
367
282
|
**Example:**
|
|
368
283
|
|
|
369
|
-
```
|
|
370
|
-
const
|
|
371
|
-
console.log(
|
|
372
|
-
console.log(
|
|
373
|
-
console.log(
|
|
284
|
+
```typescript
|
|
285
|
+
const master = await getMaster();
|
|
286
|
+
console.log(master.version);
|
|
287
|
+
console.log(master.books.length);
|
|
288
|
+
console.log(master.authors.length);
|
|
289
|
+
console.log(master.categories.length);
|
|
374
290
|
```
|
|
375
291
|
|
|
376
|
-
|
|
292
|
+
### getCoverUrl
|
|
377
293
|
|
|
378
294
|
Generates the URL for a book's cover image.
|
|
379
295
|
|
|
@@ -381,302 +297,223 @@ Generates the URL for a book's cover image.
|
|
|
381
297
|
getCoverUrl(bookId: number): string
|
|
382
298
|
```
|
|
383
299
|
|
|
384
|
-
- `bookId`:
|
|
300
|
+
- `bookId`: Book identifier
|
|
385
301
|
|
|
386
|
-
**Returns:**
|
|
302
|
+
**Returns:** Cover image URL
|
|
387
303
|
|
|
388
304
|
**Example:**
|
|
389
305
|
|
|
390
|
-
```
|
|
306
|
+
```typescript
|
|
391
307
|
const coverUrl = getCoverUrl(26592);
|
|
392
|
-
|
|
308
|
+
// Returns: "https://shamela.ws/covers/26592.jpg"
|
|
393
309
|
```
|
|
394
310
|
|
|
395
311
|
## Examples
|
|
396
312
|
|
|
397
313
|
### Downloading the Master Database
|
|
398
314
|
|
|
399
|
-
```
|
|
315
|
+
```typescript
|
|
400
316
|
import { downloadMasterDatabase } from 'shamela';
|
|
401
317
|
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
outputFile: { path: './shamela_master.json' },
|
|
413
|
-
});
|
|
414
|
-
console.log(`Master data exported to: ${jsonPath}`);
|
|
415
|
-
console.log('The JSON file includes authors, books, categories, and the master version number.');
|
|
416
|
-
} catch (error) {
|
|
417
|
-
console.error('Error downloading master database:', error);
|
|
418
|
-
}
|
|
419
|
-
})();
|
|
318
|
+
// Download as SQLite
|
|
319
|
+
const dbPath = await downloadMasterDatabase({
|
|
320
|
+
outputFile: { path: './shamela_master.db' }
|
|
321
|
+
});
|
|
322
|
+
console.log(`Downloaded to: ${dbPath}`);
|
|
323
|
+
|
|
324
|
+
// Download as JSON
|
|
325
|
+
const jsonPath = await downloadMasterDatabase({
|
|
326
|
+
outputFile: { path: './shamela_master.json' }
|
|
327
|
+
});
|
|
420
328
|
```
|
|
421
329
|
|
|
422
330
|
### Downloading a Book
|
|
423
331
|
|
|
424
|
-
```
|
|
332
|
+
```typescript
|
|
425
333
|
import { downloadBook, getBookMetadata } from 'shamela';
|
|
426
334
|
|
|
427
|
-
|
|
428
|
-
const bookId = 26592;
|
|
335
|
+
const bookId = 26592;
|
|
429
336
|
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
});
|
|
337
|
+
// Download book
|
|
338
|
+
await downloadBook(bookId, {
|
|
339
|
+
outputFile: { path: `./book_${bookId}.db` }
|
|
340
|
+
});
|
|
435
341
|
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
} catch (error) {
|
|
443
|
-
console.error('Error downloading book:', error);
|
|
444
|
-
}
|
|
445
|
-
})();
|
|
342
|
+
// With pre-fetched metadata
|
|
343
|
+
const metadata = await getBookMetadata(bookId);
|
|
344
|
+
await downloadBook(bookId, {
|
|
345
|
+
bookMetadata: metadata,
|
|
346
|
+
outputFile: { path: `./book_${bookId}.json` }
|
|
347
|
+
});
|
|
446
348
|
```
|
|
447
349
|
|
|
448
350
|
### Retrieving Book Data
|
|
449
351
|
|
|
450
|
-
```
|
|
352
|
+
```typescript
|
|
451
353
|
import { getBook } from 'shamela';
|
|
452
354
|
|
|
453
|
-
|
|
454
|
-
try {
|
|
455
|
-
const bookData = await getBook(26592);
|
|
355
|
+
const book = await getBook(26592);
|
|
456
356
|
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
if (bookData.titles) {
|
|
460
|
-
console.log(`Book has ${bookData.titles.length} titles/chapters`);
|
|
461
|
-
|
|
462
|
-
// Display table of contents
|
|
463
|
-
bookData.titles.forEach((title) => {
|
|
464
|
-
console.log(`${title.id}: ${title.content} (Page ${title.page})`);
|
|
465
|
-
});
|
|
466
|
-
}
|
|
467
|
-
|
|
468
|
-
// Access page content
|
|
469
|
-
const firstPage = bookData.pages[0];
|
|
470
|
-
console.log(`First page content: ${firstPage.content.substring(0, 100)}...`);
|
|
471
|
-
} catch (error) {
|
|
472
|
-
console.error('Error retrieving book:', error);
|
|
473
|
-
}
|
|
474
|
-
})();
|
|
475
|
-
```
|
|
357
|
+
console.log(`Book has ${book.pages.length} pages`);
|
|
476
358
|
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
(async () => {
|
|
483
|
-
try {
|
|
484
|
-
const masterData = await getMaster();
|
|
359
|
+
// Display table of contents
|
|
360
|
+
book.titles?.forEach(title => {
|
|
361
|
+
console.log(`${title.id}: ${title.content} (Page ${title.page})`);
|
|
362
|
+
});
|
|
485
363
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
} catch (error) {
|
|
490
|
-
console.error('Error retrieving master data:', error);
|
|
491
|
-
}
|
|
492
|
-
})();
|
|
364
|
+
// Access page content
|
|
365
|
+
const firstPage = book.pages[0];
|
|
366
|
+
console.log(firstPage.content.substring(0, 100));
|
|
493
367
|
```
|
|
494
368
|
|
|
495
|
-
### Getting Book
|
|
496
|
-
|
|
497
|
-
```javascript
|
|
498
|
-
import { getCoverUrl, downloadMasterDatabase } from 'shamela';
|
|
369
|
+
### Getting Book Covers
|
|
499
370
|
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
// Download master data to get book information
|
|
503
|
-
const masterData = await downloadMasterDatabase({
|
|
504
|
-
outputFile: { path: './master.json' },
|
|
505
|
-
});
|
|
371
|
+
```typescript
|
|
372
|
+
import { getCoverUrl, getMaster } from 'shamela';
|
|
506
373
|
|
|
507
|
-
|
|
508
|
-
const data = await Bun.file('./master.json').json();
|
|
374
|
+
const master = await getMaster();
|
|
509
375
|
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
} catch (error) {
|
|
516
|
-
console.error('Error processing covers:', error);
|
|
517
|
-
}
|
|
518
|
-
})();
|
|
376
|
+
// Generate cover URLs for all books
|
|
377
|
+
master.books.forEach(book => {
|
|
378
|
+
const coverUrl = getCoverUrl(book.id);
|
|
379
|
+
console.log(`${book.name}: ${coverUrl}`);
|
|
380
|
+
});
|
|
519
381
|
```
|
|
520
382
|
|
|
521
383
|
## Data Structures
|
|
522
384
|
|
|
523
|
-
The library provides comprehensive TypeScript types for all data structures:
|
|
524
|
-
|
|
525
385
|
### BookData
|
|
526
386
|
|
|
527
|
-
|
|
528
|
-
|
|
387
|
+
```typescript
|
|
388
|
+
type BookData = {
|
|
389
|
+
pages: Page[];
|
|
390
|
+
titles: Title[];
|
|
391
|
+
};
|
|
392
|
+
```
|
|
529
393
|
|
|
530
394
|
### MasterData
|
|
531
395
|
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
396
|
+
```typescript
|
|
397
|
+
type MasterData = {
|
|
398
|
+
authors: Author[];
|
|
399
|
+
books: Book[];
|
|
400
|
+
categories: Category[];
|
|
401
|
+
version: number;
|
|
402
|
+
};
|
|
403
|
+
```
|
|
536
404
|
|
|
537
405
|
### Page
|
|
538
406
|
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
407
|
+
```typescript
|
|
408
|
+
type Page = {
|
|
409
|
+
id: number;
|
|
410
|
+
content: string;
|
|
411
|
+
part?: string;
|
|
412
|
+
page?: number;
|
|
413
|
+
number?: string;
|
|
414
|
+
};
|
|
415
|
+
```
|
|
544
416
|
|
|
545
417
|
### Title
|
|
546
418
|
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
419
|
+
```typescript
|
|
420
|
+
type Title = {
|
|
421
|
+
id: number;
|
|
422
|
+
content: string;
|
|
423
|
+
page: number;
|
|
424
|
+
parent?: number;
|
|
425
|
+
};
|
|
426
|
+
```
|
|
554
427
|
|
|
555
|
-
|
|
556
|
-
- `sanitizePageContent(content: string)`: Removes common footnote markers and normalises ligatures from Shamela pages.
|
|
428
|
+
### Content Helpers
|
|
557
429
|
|
|
558
|
-
|
|
430
|
+
- `parseContentRobust(content: string)`: Converts Shamela page HTML into structured lines
|
|
431
|
+
- `sanitizePageContent(content: string)`: Removes footnote markers and normalizes text
|
|
432
|
+
- `splitPageBodyFromFooter(content: string)`: Separates page content from footnotes
|
|
433
|
+
- `removeArabicNumericPageMarkers(text: string)`: Removes Arabic page number markers
|
|
434
|
+
- `removeTagsExceptSpan(content: string)`: Strips HTML tags except span elements
|
|
559
435
|
|
|
560
|
-
|
|
436
|
+
## Next.js Demo
|
|
561
437
|
|
|
562
|
-
|
|
438
|
+
A minimal Next.js 16 demo application is available in the `demo/` directory.
|
|
563
439
|
|
|
564
|
-
|
|
565
|
-
SHAMELA_API_MASTER_PATCH_ENDPOINT=https://dev.shamela.ws/api/v1/patches/master
|
|
566
|
-
SHAMELA_API_BOOKS_ENDPOINT=https://dev.shamela.ws/api/v1/patches/book-updates
|
|
567
|
-
# Optional when hosting the wasm asset yourself
|
|
568
|
-
# SHAMELA_SQLJS_WASM_URL=https://example.com/sql-wasm.wasm
|
|
569
|
-
```
|
|
440
|
+
**Setup:**
|
|
570
441
|
|
|
571
|
-
|
|
442
|
+
Create `demo/.env.local`:
|
|
572
443
|
|
|
573
|
-
```
|
|
574
|
-
|
|
444
|
+
```env
|
|
445
|
+
SHAMELA_API_KEY=your_api_key
|
|
446
|
+
SHAMELA_API_MASTER_PATCH_ENDPOINT=https://SHAMELA_INSTANCE.ws/api/master_patch
|
|
447
|
+
SHAMELA_API_BOOKS_ENDPOINT=https://SHAMELA_INSTANCE.ws/api/books
|
|
575
448
|
```
|
|
576
449
|
|
|
577
|
-
|
|
450
|
+
**Run:**
|
|
578
451
|
|
|
579
452
|
```bash
|
|
580
|
-
bun run demo
|
|
581
|
-
bun run demo:
|
|
453
|
+
bun run demo # Development
|
|
454
|
+
bun run demo:build # Production build
|
|
455
|
+
bun run demo:start # Production server
|
|
582
456
|
```
|
|
583
457
|
|
|
584
|
-
|
|
458
|
+
Visit [http://localhost:3000](http://localhost:3000) to explore the API.
|
|
585
459
|
|
|
586
460
|
## Troubleshooting
|
|
587
461
|
|
|
588
|
-
### Error: "Unable to locate sql-wasm.wasm file"
|
|
462
|
+
### Error: "Unable to automatically locate sql-wasm.wasm file"
|
|
589
463
|
|
|
590
|
-
This
|
|
464
|
+
This occurs in bundled environments (Next.js, webpack, Turbopack).
|
|
591
465
|
|
|
592
|
-
**Solution:**
|
|
466
|
+
**Solution:** Add explicit configuration:
|
|
593
467
|
|
|
594
468
|
```typescript
|
|
595
|
-
import { configure
|
|
596
|
-
|
|
597
|
-
// Option 1: Use the helper (recommended)
|
|
598
|
-
configure(createNodeConfig({
|
|
599
|
-
apiKey: process.env.SHAMELA_API_KEY,
|
|
600
|
-
booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,
|
|
601
|
-
masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,
|
|
602
|
-
}));
|
|
603
|
-
|
|
604
|
-
// Option 2: Manual configuration
|
|
469
|
+
import { configure } from 'shamela';
|
|
605
470
|
import { join } from 'node:path';
|
|
606
471
|
|
|
607
472
|
configure({
|
|
608
473
|
sqlJsWasmUrl: join(process.cwd(), 'node_modules', 'sql.js', 'dist', 'sql-wasm.wasm'),
|
|
609
474
|
apiKey: process.env.SHAMELA_API_KEY,
|
|
610
|
-
|
|
475
|
+
booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,
|
|
476
|
+
masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,
|
|
611
477
|
});
|
|
612
478
|
```
|
|
613
479
|
|
|
614
|
-
###
|
|
615
|
-
|
|
616
|
-
This means you're in a Node.js environment but providing an HTTPS URL for the WASM file. Node.js requires a filesystem path, not a URL.
|
|
480
|
+
### Next.js: Module not found errors
|
|
617
481
|
|
|
618
|
-
|
|
482
|
+
1. Add to `next.config.ts`:
|
|
483
|
+
```typescript
|
|
484
|
+
serverExternalPackages: ['shamela', 'sql.js']
|
|
485
|
+
```
|
|
619
486
|
|
|
620
|
-
|
|
621
|
-
// ❌ Wrong - HTTPS URL in Node.js
|
|
622
|
-
configure({
|
|
623
|
-
sqlJsWasmUrl: 'https://cdn.jsdelivr.net/npm/sql.js@1.13.0/dist/sql-wasm.wasm'
|
|
624
|
-
});
|
|
625
|
-
|
|
626
|
-
// ✅ Correct - Filesystem path or use createNodeConfig
|
|
627
|
-
import { createNodeConfig } from 'shamela';
|
|
628
|
-
|
|
629
|
-
configure(createNodeConfig({
|
|
630
|
-
apiKey: process.env.SHAMELA_API_KEY,
|
|
631
|
-
// ... other config
|
|
632
|
-
}));
|
|
633
|
-
```
|
|
487
|
+
2. Only import shamela in server-side code
|
|
634
488
|
|
|
635
|
-
|
|
489
|
+
3. Create a separate `lib/shamela-server.ts` for configuration
|
|
636
490
|
|
|
637
|
-
|
|
491
|
+
### Production build works differently than development
|
|
638
492
|
|
|
639
|
-
|
|
640
|
-
2. Ensure you're only importing shamela in server-side code (Server Actions, API Routes, Server Components)
|
|
641
|
-
3. Never import shamela in `layout.tsx` or client components
|
|
642
|
-
4. Create a separate `lib/shamela-server.ts` file for configuration
|
|
493
|
+
Ensure `serverExternalPackages` is set in your Next.js config for both development and production.
|
|
643
494
|
|
|
644
|
-
###
|
|
495
|
+
### Monorepo setup issues
|
|
645
496
|
|
|
646
|
-
|
|
497
|
+
Adjust the WASM path based on your structure:
|
|
647
498
|
|
|
648
499
|
```typescript
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,
|
|
654
|
-
masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,
|
|
655
|
-
}));
|
|
500
|
+
configure({
|
|
501
|
+
sqlJsWasmUrl: join(process.cwd(), '../../node_modules/sql.js/dist/sql-wasm.wasm'),
|
|
502
|
+
// ... other config
|
|
503
|
+
});
|
|
656
504
|
```
|
|
657
505
|
|
|
658
506
|
## Testing
|
|
659
507
|
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
```bash
|
|
663
|
-
bun test src
|
|
664
|
-
```
|
|
665
|
-
|
|
666
|
-
For end-to-end tests:
|
|
667
|
-
|
|
668
|
-
```bash
|
|
669
|
-
bun run e2e
|
|
670
|
-
```
|
|
671
|
-
|
|
672
|
-
### Formatting
|
|
673
|
-
|
|
674
|
-
Apply Biome formatting across the repository with:
|
|
508
|
+
Run tests with Bun:
|
|
675
509
|
|
|
676
510
|
```bash
|
|
677
|
-
bun
|
|
511
|
+
bun test src # Unit tests
|
|
512
|
+
bun run e2e # End-to-end tests
|
|
513
|
+
bun run format # Format code
|
|
514
|
+
bun run lint # Lint code
|
|
678
515
|
```
|
|
679
516
|
|
|
680
517
|
## License
|
|
681
518
|
|
|
682
|
-
|
|
519
|
+
MIT License - see LICENSE file for details.
|