shamela 1.3.0 → 1.3.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +280 -288
- package/dist/index.js +5 -4
- package/dist/index.js.map +1 -1
- package/package.json +6 -6
package/README.md
CHANGED
|
@@ -18,115 +18,137 @@ A universal TypeScript library for accessing and downloading Maktabah Shamela v4
|
|
|
18
18
|
## Table of Contents
|
|
19
19
|
|
|
20
20
|
- [Installation](#installation)
|
|
21
|
-
- [
|
|
22
|
-
- [
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
21
|
+
- [Quick Start](#quick-start)
|
|
22
|
+
- [Standard Node.js](#standard-nodejs)
|
|
23
|
+
- [Next.js / Bundled Environments](#nextjs--bundled-environments)
|
|
24
|
+
- [Browser](#browser)
|
|
25
|
+
- [API Reference](#api-reference)
|
|
26
|
+
- [getMasterMetadata](#getmastermetadata)
|
|
27
|
+
- [downloadMasterDatabase](#downloadmasterdatabase)
|
|
28
|
+
- [getBookMetadata](#getbookmetadata)
|
|
29
|
+
- [downloadBook](#downloadbook)
|
|
30
|
+
- [getBook](#getbook)
|
|
31
|
+
- [getMaster](#getmaster)
|
|
32
|
+
- [getCoverUrl](#getcoverurl)
|
|
32
33
|
- [Examples](#examples)
|
|
33
|
-
- [Downloading the Master Database](#downloading-the-master-database)
|
|
34
|
-
- [Downloading a Book](#downloading-a-book)
|
|
35
|
-
- [Retrieving Book Data](#retrieving-book-data)
|
|
36
|
-
- [Retrieving Master Data in memory](#retrieving-master-data-in-memory)
|
|
37
34
|
- [Data Structures](#data-structures)
|
|
38
|
-
- [Next.js
|
|
35
|
+
- [Next.js Demo](#nextjs-demo)
|
|
36
|
+
- [Troubleshooting](#troubleshooting)
|
|
39
37
|
- [Testing](#testing)
|
|
40
38
|
- [License](#license)
|
|
41
39
|
|
|
42
40
|
## Installation
|
|
43
41
|
|
|
44
42
|
```bash
|
|
45
|
-
|
|
43
|
+
npm install shamela
|
|
46
44
|
```
|
|
47
45
|
|
|
48
|
-
or
|
|
49
|
-
|
|
50
46
|
```bash
|
|
51
|
-
|
|
47
|
+
bun add shamela
|
|
52
48
|
```
|
|
53
49
|
|
|
54
|
-
or
|
|
55
|
-
|
|
56
50
|
```bash
|
|
57
51
|
yarn add shamela
|
|
58
52
|
```
|
|
59
53
|
|
|
60
|
-
or
|
|
61
|
-
|
|
62
54
|
```bash
|
|
63
55
|
pnpm install shamela
|
|
64
56
|
```
|
|
65
57
|
|
|
66
|
-
##
|
|
58
|
+
## Quick Start
|
|
59
|
+
|
|
60
|
+
### Standard Node.js
|
|
67
61
|
|
|
68
|
-
|
|
62
|
+
For simple Node.js scripts (non-bundled environments), the library auto-detects the WASM file:
|
|
69
63
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
- `SHAMELA_API_BOOKS_ENDPOINT`: The base endpoint URL for book-related API calls.
|
|
73
|
-
- `SHAMELA_SQLJS_WASM_URL` (optional): Override the default CDN URL used to load the `sql.js` WebAssembly binary when running in the browser.
|
|
64
|
+
```typescript
|
|
65
|
+
import { configure, getBook } from 'shamela';
|
|
74
66
|
|
|
75
|
-
|
|
67
|
+
// Configure API credentials
|
|
68
|
+
configure({
|
|
69
|
+
apiKey: process.env.SHAMELA_API_KEY,
|
|
70
|
+
booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,
|
|
71
|
+
masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,
|
|
72
|
+
// sqlJsWasmUrl is auto-detected in standard Node.js
|
|
73
|
+
});
|
|
76
74
|
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
SHAMELA_API_BOOKS_ENDPOINT=https://shamela.ws/api/books
|
|
81
|
-
# Optional when you host sql-wasm.wasm yourself
|
|
82
|
-
# SHAMELA_SQLJS_WASM_URL=https://example.com/sql-wasm.wasm
|
|
75
|
+
// Use the library
|
|
76
|
+
const book = await getBook(26592);
|
|
77
|
+
console.log(`Downloaded book with ${book.pages.length} pages`);
|
|
83
78
|
```
|
|
84
79
|
|
|
85
|
-
###
|
|
80
|
+
### Next.js / Bundled Environments
|
|
81
|
+
|
|
82
|
+
For Next.js, webpack, Turbopack, and other bundlers, you need to explicitly configure the WASM file path.
|
|
86
83
|
|
|
87
|
-
|
|
84
|
+
**1. Update `next.config.ts` or `next.config.js`:**
|
|
88
85
|
|
|
89
|
-
```
|
|
86
|
+
```typescript
|
|
87
|
+
import type { NextConfig } from 'next';
|
|
88
|
+
|
|
89
|
+
const nextConfig: NextConfig = {
|
|
90
|
+
serverExternalPackages: ['shamela', 'sql.js'],
|
|
91
|
+
// ... rest of your config
|
|
92
|
+
};
|
|
93
|
+
|
|
94
|
+
export default nextConfig;
|
|
95
|
+
```
|
|
96
|
+
|
|
97
|
+
**2. Create a server-only configuration file:**
|
|
98
|
+
|
|
99
|
+
```typescript
|
|
100
|
+
// lib/shamela-server.ts
|
|
90
101
|
import { configure } from 'shamela';
|
|
102
|
+
import { join } from 'node:path';
|
|
91
103
|
|
|
92
104
|
configure({
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
sqlJsWasmUrl: '/assets/sql-wasm.wasm',
|
|
98
|
-
// Optional: integrate with your application's logging system
|
|
99
|
-
logger: console,
|
|
100
|
-
// Optional: provide a custom fetch implementation (for tests or SSR)
|
|
101
|
-
fetchImplementation: fetch,
|
|
105
|
+
sqlJsWasmUrl: join(process.cwd(), 'node_modules', 'sql.js', 'dist', 'sql-wasm.wasm'),
|
|
106
|
+
apiKey: process.env.SHAMELA_API_KEY,
|
|
107
|
+
booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,
|
|
108
|
+
masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,
|
|
102
109
|
});
|
|
110
|
+
|
|
111
|
+
export { downloadBook, getBook, getBookMetadata, getMaster, downloadMasterDatabase } from 'shamela';
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
**3. Use in Server Actions:**
|
|
115
|
+
|
|
116
|
+
```typescript
|
|
117
|
+
'use server';
|
|
118
|
+
|
|
119
|
+
import { getBookMetadata, downloadBook } from '@/lib/shamela-server';
|
|
120
|
+
|
|
121
|
+
export async function downloadBookAction(bookId: number) {
|
|
122
|
+
const metadata = await getBookMetadata(bookId);
|
|
123
|
+
return await downloadBook(bookId, {
|
|
124
|
+
bookMetadata: metadata,
|
|
125
|
+
outputFile: { path: `./books/${bookId}.db` }
|
|
126
|
+
});
|
|
127
|
+
}
|
|
103
128
|
```
|
|
104
129
|
|
|
105
|
-
|
|
130
|
+
**Important:** Only import `shamela` in server-side code (Server Actions, API Routes, or Server Components). Never import in client components or `layout.tsx`.
|
|
106
131
|
|
|
107
|
-
|
|
132
|
+
### Browser
|
|
108
133
|
|
|
109
|
-
|
|
134
|
+
In browsers, the library automatically uses a CDN-hosted WASM file:
|
|
110
135
|
|
|
111
|
-
|
|
136
|
+
```typescript
|
|
137
|
+
import { configure, getBook } from 'shamela';
|
|
112
138
|
|
|
113
|
-
|
|
139
|
+
configure({
|
|
140
|
+
apiKey: 'your-api-key',
|
|
141
|
+
booksEndpoint: 'https://SHAMELA_INSTANCE.ws/api/books',
|
|
142
|
+
masterPatchEndpoint: 'https://SHAMELA_INSTANCE.ws/api/master_patch',
|
|
143
|
+
// Automatically uses CDN: https://cdn.jsdelivr.net/npm/sql.js@1.13.0/dist/sql-wasm.wasm
|
|
144
|
+
});
|
|
114
145
|
|
|
115
|
-
|
|
116
|
-
import {
|
|
117
|
-
getMasterMetadata,
|
|
118
|
-
downloadMasterDatabase,
|
|
119
|
-
getBookMetadata,
|
|
120
|
-
downloadBook,
|
|
121
|
-
getBook,
|
|
122
|
-
getMaster,
|
|
123
|
-
getCoverUrl,
|
|
124
|
-
} from 'shamela';
|
|
146
|
+
const book = await getBook(26592);
|
|
125
147
|
```
|
|
126
148
|
|
|
127
|
-
|
|
149
|
+
## API Reference
|
|
128
150
|
|
|
129
|
-
|
|
151
|
+
### getMasterMetadata
|
|
130
152
|
|
|
131
153
|
Fetches metadata for the master database.
|
|
132
154
|
|
|
@@ -134,57 +156,49 @@ Fetches metadata for the master database.
|
|
|
134
156
|
getMasterMetadata(version?: number): Promise<GetMasterMetadataResponsePayload>
|
|
135
157
|
```
|
|
136
158
|
|
|
137
|
-
- `version` (optional): The version number
|
|
159
|
+
- `version` (optional): The version number to check for updates (defaults to 0)
|
|
138
160
|
|
|
139
|
-
**Returns:** Promise
|
|
161
|
+
**Returns:** Promise resolving to master database metadata including download URL and version
|
|
140
162
|
|
|
141
163
|
**Example:**
|
|
142
164
|
|
|
143
|
-
```
|
|
144
|
-
const
|
|
145
|
-
console.log(
|
|
146
|
-
console.log(
|
|
165
|
+
```typescript
|
|
166
|
+
const metadata = await getMasterMetadata();
|
|
167
|
+
console.log(metadata.url); // Download URL
|
|
168
|
+
console.log(metadata.version); // Version number
|
|
147
169
|
|
|
148
170
|
// Check for updates from a specific version
|
|
149
171
|
const updates = await getMasterMetadata(5);
|
|
150
172
|
```
|
|
151
173
|
|
|
152
|
-
|
|
174
|
+
### downloadMasterDatabase
|
|
153
175
|
|
|
154
|
-
Downloads the master database
|
|
176
|
+
Downloads the master database containing all books, authors, and categories.
|
|
155
177
|
|
|
156
178
|
```typescript
|
|
157
179
|
downloadMasterDatabase(options: DownloadMasterOptions): Promise<string>
|
|
158
180
|
```
|
|
159
181
|
|
|
160
|
-
- `options
|
|
161
|
-
|
|
162
|
-
- `outputFile`: Object with `path` property specifying the output file path
|
|
182
|
+
- `options.masterMetadata` (optional): Pre-fetched metadata
|
|
183
|
+
- `options.outputFile.path`: Output file path (`.db`, `.sqlite`, or `.json`)
|
|
163
184
|
|
|
164
|
-
**Returns:** Promise
|
|
185
|
+
**Returns:** Promise resolving to the output file path
|
|
165
186
|
|
|
166
187
|
**Example:**
|
|
167
188
|
|
|
168
|
-
```
|
|
189
|
+
```typescript
|
|
169
190
|
// Download as SQLite database
|
|
170
191
|
await downloadMasterDatabase({
|
|
171
|
-
|
|
192
|
+
outputFile: { path: './master.db' }
|
|
172
193
|
});
|
|
173
194
|
|
|
174
195
|
// Download as JSON
|
|
175
196
|
await downloadMasterDatabase({
|
|
176
|
-
|
|
177
|
-
});
|
|
178
|
-
|
|
179
|
-
// Use pre-fetched metadata for efficiency
|
|
180
|
-
const masterMetadata = await getMasterMetadata();
|
|
181
|
-
await downloadMasterDatabase({
|
|
182
|
-
masterMetadata,
|
|
183
|
-
outputFile: { path: './master.db' },
|
|
197
|
+
outputFile: { path: './master.json' }
|
|
184
198
|
});
|
|
185
199
|
```
|
|
186
200
|
|
|
187
|
-
|
|
201
|
+
### getBookMetadata
|
|
188
202
|
|
|
189
203
|
Fetches metadata for a specific book.
|
|
190
204
|
|
|
@@ -192,105 +206,90 @@ Fetches metadata for a specific book.
|
|
|
192
206
|
getBookMetadata(id: number, options?: GetBookMetadataOptions): Promise<GetBookMetadataResponsePayload>
|
|
193
207
|
```
|
|
194
208
|
|
|
195
|
-
- `id`:
|
|
196
|
-
- `options` (optional):
|
|
197
|
-
|
|
198
|
-
- `minorVersion`: The minor version to check against
|
|
209
|
+
- `id`: Book identifier
|
|
210
|
+
- `options.majorVersion` (optional): Major version to check
|
|
211
|
+
- `options.minorVersion` (optional): Minor version to check
|
|
199
212
|
|
|
200
|
-
**Returns:** Promise
|
|
213
|
+
**Returns:** Promise resolving to book metadata
|
|
201
214
|
|
|
202
215
|
**Example:**
|
|
203
216
|
|
|
204
|
-
```
|
|
217
|
+
```typescript
|
|
205
218
|
const metadata = await getBookMetadata(26592);
|
|
206
|
-
console.log(metadata.majorReleaseUrl);
|
|
207
|
-
console.log(metadata.
|
|
208
|
-
|
|
209
|
-
// Check specific versions
|
|
210
|
-
const versionedMetadata = await getBookMetadata(26592, {
|
|
211
|
-
majorVersion: 1,
|
|
212
|
-
minorVersion: 2,
|
|
213
|
-
});
|
|
219
|
+
console.log(metadata.majorReleaseUrl);
|
|
220
|
+
console.log(metadata.minorReleaseUrl);
|
|
214
221
|
```
|
|
215
222
|
|
|
216
|
-
|
|
223
|
+
### downloadBook
|
|
217
224
|
|
|
218
|
-
Downloads and processes a book from
|
|
225
|
+
Downloads and processes a book from Shamela.
|
|
219
226
|
|
|
220
227
|
```typescript
|
|
221
228
|
downloadBook(id: number, options: DownloadBookOptions): Promise<string>
|
|
222
229
|
```
|
|
223
230
|
|
|
224
|
-
- `id`:
|
|
225
|
-
- `options
|
|
226
|
-
|
|
227
|
-
- `outputFile`: Object with `path` property specifying the output file path
|
|
231
|
+
- `id`: Book identifier
|
|
232
|
+
- `options.bookMetadata` (optional): Pre-fetched metadata
|
|
233
|
+
- `options.outputFile.path`: Output file path (`.db`, `.sqlite`, or `.json`)
|
|
228
234
|
|
|
229
|
-
**Returns:** Promise
|
|
235
|
+
**Returns:** Promise resolving to the output file path
|
|
230
236
|
|
|
231
237
|
**Example:**
|
|
232
238
|
|
|
233
|
-
```
|
|
239
|
+
```typescript
|
|
234
240
|
// Download as JSON
|
|
235
241
|
await downloadBook(26592, {
|
|
236
|
-
|
|
237
|
-
});
|
|
238
|
-
|
|
239
|
-
// Download as SQLite database
|
|
240
|
-
await downloadBook(26592, {
|
|
241
|
-
outputFile: { path: './book.db' },
|
|
242
|
+
outputFile: { path: './book.json' }
|
|
242
243
|
});
|
|
243
244
|
|
|
244
|
-
//
|
|
245
|
-
const bookMetadata = await getBookMetadata(26592);
|
|
245
|
+
// Download as SQLite
|
|
246
246
|
await downloadBook(26592, {
|
|
247
|
-
|
|
248
|
-
outputFile: { path: './book.db' },
|
|
247
|
+
outputFile: { path: './book.db' }
|
|
249
248
|
});
|
|
250
249
|
```
|
|
251
250
|
|
|
252
|
-
|
|
251
|
+
### getBook
|
|
253
252
|
|
|
254
|
-
Retrieves complete book data as a JavaScript object.
|
|
253
|
+
Retrieves complete book data as a JavaScript object.
|
|
255
254
|
|
|
256
255
|
```typescript
|
|
257
256
|
getBook(id: number): Promise<BookData>
|
|
258
257
|
```
|
|
259
258
|
|
|
260
|
-
- `id`:
|
|
259
|
+
- `id`: Book identifier
|
|
261
260
|
|
|
262
|
-
**Returns:** Promise
|
|
261
|
+
**Returns:** Promise resolving to book data with pages and titles
|
|
263
262
|
|
|
264
263
|
**Example:**
|
|
265
264
|
|
|
266
|
-
```
|
|
267
|
-
const
|
|
268
|
-
console.log(
|
|
269
|
-
console.log(
|
|
270
|
-
console.log(
|
|
265
|
+
```typescript
|
|
266
|
+
const book = await getBook(26592);
|
|
267
|
+
console.log(book.pages.length);
|
|
268
|
+
console.log(book.titles?.length);
|
|
269
|
+
console.log(book.pages[0].content);
|
|
271
270
|
```
|
|
272
271
|
|
|
273
|
-
|
|
272
|
+
### getMaster
|
|
274
273
|
|
|
275
|
-
Retrieves the entire master dataset
|
|
276
|
-
API reports for the snapshot.
|
|
274
|
+
Retrieves the entire master dataset as a JavaScript object.
|
|
277
275
|
|
|
278
276
|
```typescript
|
|
279
277
|
getMaster(): Promise<MasterData>
|
|
280
278
|
```
|
|
281
279
|
|
|
282
|
-
**Returns:** Promise
|
|
280
|
+
**Returns:** Promise resolving to master data with authors, books, categories, and version
|
|
283
281
|
|
|
284
282
|
**Example:**
|
|
285
283
|
|
|
286
|
-
```
|
|
287
|
-
const
|
|
288
|
-
console.log(
|
|
289
|
-
console.log(
|
|
290
|
-
console.log(
|
|
284
|
+
```typescript
|
|
285
|
+
const master = await getMaster();
|
|
286
|
+
console.log(master.version);
|
|
287
|
+
console.log(master.books.length);
|
|
288
|
+
console.log(master.authors.length);
|
|
289
|
+
console.log(master.categories.length);
|
|
291
290
|
```
|
|
292
291
|
|
|
293
|
-
|
|
292
|
+
### getCoverUrl
|
|
294
293
|
|
|
295
294
|
Generates the URL for a book's cover image.
|
|
296
295
|
|
|
@@ -298,230 +297,223 @@ Generates the URL for a book's cover image.
|
|
|
298
297
|
getCoverUrl(bookId: number): string
|
|
299
298
|
```
|
|
300
299
|
|
|
301
|
-
- `bookId`:
|
|
300
|
+
- `bookId`: Book identifier
|
|
302
301
|
|
|
303
|
-
**Returns:**
|
|
302
|
+
**Returns:** Cover image URL
|
|
304
303
|
|
|
305
304
|
**Example:**
|
|
306
305
|
|
|
307
|
-
```
|
|
306
|
+
```typescript
|
|
308
307
|
const coverUrl = getCoverUrl(26592);
|
|
309
|
-
|
|
308
|
+
// Returns: "https://shamela.ws/covers/26592.jpg"
|
|
310
309
|
```
|
|
311
310
|
|
|
312
311
|
## Examples
|
|
313
312
|
|
|
314
313
|
### Downloading the Master Database
|
|
315
314
|
|
|
316
|
-
```
|
|
315
|
+
```typescript
|
|
317
316
|
import { downloadMasterDatabase } from 'shamela';
|
|
318
317
|
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
outputFile: { path: './shamela_master.json' },
|
|
330
|
-
});
|
|
331
|
-
console.log(`Master data exported to: ${jsonPath}`);
|
|
332
|
-
console.log('The JSON file includes authors, books, categories, and the master version number.');
|
|
333
|
-
} catch (error) {
|
|
334
|
-
console.error('Error downloading master database:', error);
|
|
335
|
-
}
|
|
336
|
-
})();
|
|
318
|
+
// Download as SQLite
|
|
319
|
+
const dbPath = await downloadMasterDatabase({
|
|
320
|
+
outputFile: { path: './shamela_master.db' }
|
|
321
|
+
});
|
|
322
|
+
console.log(`Downloaded to: ${dbPath}`);
|
|
323
|
+
|
|
324
|
+
// Download as JSON
|
|
325
|
+
const jsonPath = await downloadMasterDatabase({
|
|
326
|
+
outputFile: { path: './shamela_master.json' }
|
|
327
|
+
});
|
|
337
328
|
```
|
|
338
329
|
|
|
339
330
|
### Downloading a Book
|
|
340
331
|
|
|
341
|
-
```
|
|
332
|
+
```typescript
|
|
342
333
|
import { downloadBook, getBookMetadata } from 'shamela';
|
|
343
334
|
|
|
344
|
-
|
|
345
|
-
const bookId = 26592;
|
|
335
|
+
const bookId = 26592;
|
|
346
336
|
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
});
|
|
337
|
+
// Download book
|
|
338
|
+
await downloadBook(bookId, {
|
|
339
|
+
outputFile: { path: `./book_${bookId}.db` }
|
|
340
|
+
});
|
|
352
341
|
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
} catch (error) {
|
|
360
|
-
console.error('Error downloading book:', error);
|
|
361
|
-
}
|
|
362
|
-
})();
|
|
342
|
+
// With pre-fetched metadata
|
|
343
|
+
const metadata = await getBookMetadata(bookId);
|
|
344
|
+
await downloadBook(bookId, {
|
|
345
|
+
bookMetadata: metadata,
|
|
346
|
+
outputFile: { path: `./book_${bookId}.json` }
|
|
347
|
+
});
|
|
363
348
|
```
|
|
364
349
|
|
|
365
350
|
### Retrieving Book Data
|
|
366
351
|
|
|
367
|
-
```
|
|
352
|
+
```typescript
|
|
368
353
|
import { getBook } from 'shamela';
|
|
369
354
|
|
|
370
|
-
|
|
371
|
-
try {
|
|
372
|
-
const bookData = await getBook(26592);
|
|
373
|
-
|
|
374
|
-
console.log(`Book has ${bookData.pages.length} pages`);
|
|
375
|
-
|
|
376
|
-
if (bookData.titles) {
|
|
377
|
-
console.log(`Book has ${bookData.titles.length} titles/chapters`);
|
|
378
|
-
|
|
379
|
-
// Display table of contents
|
|
380
|
-
bookData.titles.forEach((title) => {
|
|
381
|
-
console.log(`${title.id}: ${title.content} (Page ${title.page})`);
|
|
382
|
-
});
|
|
383
|
-
}
|
|
384
|
-
|
|
385
|
-
// Access page content
|
|
386
|
-
const firstPage = bookData.pages[0];
|
|
387
|
-
console.log(`First page content: ${firstPage.content.substring(0, 100)}...`);
|
|
388
|
-
} catch (error) {
|
|
389
|
-
console.error('Error retrieving book:', error);
|
|
390
|
-
}
|
|
391
|
-
})();
|
|
392
|
-
```
|
|
393
|
-
|
|
394
|
-
### Retrieving Master Data in memory
|
|
355
|
+
const book = await getBook(26592);
|
|
395
356
|
|
|
396
|
-
|
|
397
|
-
import { getMaster } from 'shamela';
|
|
357
|
+
console.log(`Book has ${book.pages.length} pages`);
|
|
398
358
|
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
359
|
+
// Display table of contents
|
|
360
|
+
book.titles?.forEach(title => {
|
|
361
|
+
console.log(`${title.id}: ${title.content} (Page ${title.page})`);
|
|
362
|
+
});
|
|
402
363
|
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
} catch (error) {
|
|
407
|
-
console.error('Error retrieving master data:', error);
|
|
408
|
-
}
|
|
409
|
-
})();
|
|
364
|
+
// Access page content
|
|
365
|
+
const firstPage = book.pages[0];
|
|
366
|
+
console.log(firstPage.content.substring(0, 100));
|
|
410
367
|
```
|
|
411
368
|
|
|
412
|
-
### Getting Book
|
|
413
|
-
|
|
414
|
-
```javascript
|
|
415
|
-
import { getCoverUrl, downloadMasterDatabase } from 'shamela';
|
|
369
|
+
### Getting Book Covers
|
|
416
370
|
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
// Download master data to get book information
|
|
420
|
-
const masterData = await downloadMasterDatabase({
|
|
421
|
-
outputFile: { path: './master.json' },
|
|
422
|
-
});
|
|
371
|
+
```typescript
|
|
372
|
+
import { getCoverUrl, getMaster } from 'shamela';
|
|
423
373
|
|
|
424
|
-
|
|
425
|
-
const data = await Bun.file('./master.json').json();
|
|
374
|
+
const master = await getMaster();
|
|
426
375
|
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
} catch (error) {
|
|
433
|
-
console.error('Error processing covers:', error);
|
|
434
|
-
}
|
|
435
|
-
})();
|
|
376
|
+
// Generate cover URLs for all books
|
|
377
|
+
master.books.forEach(book => {
|
|
378
|
+
const coverUrl = getCoverUrl(book.id);
|
|
379
|
+
console.log(`${book.name}: ${coverUrl}`);
|
|
380
|
+
});
|
|
436
381
|
```
|
|
437
382
|
|
|
438
383
|
## Data Structures
|
|
439
384
|
|
|
440
|
-
The library provides comprehensive TypeScript types for all data structures:
|
|
441
|
-
|
|
442
385
|
### BookData
|
|
443
386
|
|
|
444
|
-
|
|
445
|
-
|
|
387
|
+
```typescript
|
|
388
|
+
type BookData = {
|
|
389
|
+
pages: Page[];
|
|
390
|
+
titles: Title[];
|
|
391
|
+
};
|
|
392
|
+
```
|
|
446
393
|
|
|
447
394
|
### MasterData
|
|
448
395
|
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
396
|
+
```typescript
|
|
397
|
+
type MasterData = {
|
|
398
|
+
authors: Author[];
|
|
399
|
+
books: Book[];
|
|
400
|
+
categories: Category[];
|
|
401
|
+
version: number;
|
|
402
|
+
};
|
|
403
|
+
```
|
|
453
404
|
|
|
454
405
|
### Page
|
|
455
406
|
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
407
|
+
```typescript
|
|
408
|
+
type Page = {
|
|
409
|
+
id: number;
|
|
410
|
+
content: string;
|
|
411
|
+
part?: string;
|
|
412
|
+
page?: number;
|
|
413
|
+
number?: string;
|
|
414
|
+
};
|
|
415
|
+
```
|
|
461
416
|
|
|
462
417
|
### Title
|
|
463
418
|
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
419
|
+
```typescript
|
|
420
|
+
type Title = {
|
|
421
|
+
id: number;
|
|
422
|
+
content: string;
|
|
423
|
+
page: number;
|
|
424
|
+
parent?: number;
|
|
425
|
+
};
|
|
426
|
+
```
|
|
427
|
+
|
|
428
|
+
### Content Helpers
|
|
469
429
|
|
|
470
|
-
|
|
430
|
+
- `parseContentRobust(content: string)`: Converts Shamela page HTML into structured lines
|
|
431
|
+
- `sanitizePageContent(content: string)`: Removes footnote markers and normalizes text
|
|
432
|
+
- `splitPageBodyFromFooter(content: string)`: Separates page content from footnotes
|
|
433
|
+
- `removeArabicNumericPageMarkers(text: string)`: Removes Arabic page number markers
|
|
434
|
+
- `removeTagsExceptSpan(content: string)`: Strips HTML tags except span elements
|
|
471
435
|
|
|
472
|
-
|
|
473
|
-
- `sanitizePageContent(content: string)`: Removes common footnote markers and normalises ligatures from Shamela pages.
|
|
436
|
+
## Next.js Demo
|
|
474
437
|
|
|
475
|
-
|
|
438
|
+
A minimal Next.js 16 demo application is available in the `demo/` directory.
|
|
476
439
|
|
|
477
|
-
|
|
440
|
+
**Setup:**
|
|
478
441
|
|
|
479
|
-
Create
|
|
442
|
+
Create `demo/.env.local`:
|
|
480
443
|
|
|
481
|
-
```
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
# SHAMELA_SQLJS_WASM_URL=https://example.com/sql-wasm.wasm
|
|
444
|
+
```env
|
|
445
|
+
SHAMELA_API_KEY=your_api_key
|
|
446
|
+
SHAMELA_API_MASTER_PATCH_ENDPOINT=https://SHAMELA_INSTANCE.ws/api/master_patch
|
|
447
|
+
SHAMELA_API_BOOKS_ENDPOINT=https://SHAMELA_INSTANCE.ws/api/books
|
|
486
448
|
```
|
|
487
449
|
|
|
488
|
-
|
|
450
|
+
**Run:**
|
|
489
451
|
|
|
490
452
|
```bash
|
|
491
|
-
bun run demo
|
|
453
|
+
bun run demo # Development
|
|
454
|
+
bun run demo:build # Production build
|
|
455
|
+
bun run demo:start # Production server
|
|
492
456
|
```
|
|
493
457
|
|
|
494
|
-
Visit [http://localhost:3000](http://localhost:3000) to
|
|
458
|
+
Visit [http://localhost:3000](http://localhost:3000) to explore the API.
|
|
495
459
|
|
|
496
|
-
|
|
497
|
-
bun run demo:build
|
|
498
|
-
bun run demo:start
|
|
499
|
-
```
|
|
460
|
+
## Troubleshooting
|
|
500
461
|
|
|
501
|
-
|
|
462
|
+
### Error: "Unable to automatically locate sql-wasm.wasm file"
|
|
502
463
|
|
|
503
|
-
|
|
464
|
+
This occurs in bundled environments (Next.js, webpack, Turbopack).
|
|
504
465
|
|
|
505
|
-
|
|
466
|
+
**Solution:** Add explicit configuration:
|
|
506
467
|
|
|
507
|
-
```
|
|
508
|
-
|
|
468
|
+
```typescript
|
|
469
|
+
import { configure } from 'shamela';
|
|
470
|
+
import { join } from 'node:path';
|
|
471
|
+
|
|
472
|
+
configure({
|
|
473
|
+
sqlJsWasmUrl: join(process.cwd(), 'node_modules', 'sql.js', 'dist', 'sql-wasm.wasm'),
|
|
474
|
+
apiKey: process.env.SHAMELA_API_KEY,
|
|
475
|
+
booksEndpoint: process.env.SHAMELA_BOOKS_ENDPOINT,
|
|
476
|
+
masterPatchEndpoint: process.env.SHAMELA_MASTER_ENDPOINT,
|
|
477
|
+
});
|
|
509
478
|
```
|
|
510
479
|
|
|
511
|
-
|
|
480
|
+
### Next.js: Module not found errors
|
|
512
481
|
|
|
513
|
-
|
|
514
|
-
|
|
482
|
+
1. Add to `next.config.ts`:
|
|
483
|
+
```typescript
|
|
484
|
+
serverExternalPackages: ['shamela', 'sql.js']
|
|
485
|
+
```
|
|
486
|
+
|
|
487
|
+
2. Only import shamela in server-side code
|
|
488
|
+
|
|
489
|
+
3. Create a separate `lib/shamela-server.ts` for configuration
|
|
490
|
+
|
|
491
|
+
### Production build works differently than development
|
|
492
|
+
|
|
493
|
+
Ensure `serverExternalPackages` is set in your Next.js config for both development and production.
|
|
494
|
+
|
|
495
|
+
### Monorepo setup issues
|
|
496
|
+
|
|
497
|
+
Adjust the WASM path based on your structure:
|
|
498
|
+
|
|
499
|
+
```typescript
|
|
500
|
+
configure({
|
|
501
|
+
sqlJsWasmUrl: join(process.cwd(), '../../node_modules/sql.js/dist/sql-wasm.wasm'),
|
|
502
|
+
// ... other config
|
|
503
|
+
});
|
|
515
504
|
```
|
|
516
505
|
|
|
517
|
-
|
|
506
|
+
## Testing
|
|
518
507
|
|
|
519
|
-
|
|
508
|
+
Run tests with Bun:
|
|
520
509
|
|
|
521
510
|
```bash
|
|
522
|
-
bun
|
|
511
|
+
bun test src # Unit tests
|
|
512
|
+
bun run e2e # End-to-end tests
|
|
513
|
+
bun run format # Format code
|
|
514
|
+
bun run lint # Lint code
|
|
523
515
|
```
|
|
524
516
|
|
|
525
517
|
## License
|
|
526
518
|
|
|
527
|
-
|
|
519
|
+
MIT License - see LICENSE file for details.
|