podcast-dl 11.3.0 → 11.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -21
- package/README.md +146 -122
- package/bin/archive.js +39 -39
- package/bin/async.js +36 -13
- package/bin/bin.js +2 -7
- package/bin/commander.js +199 -199
- package/bin/exec.js +30 -30
- package/bin/ffmpeg.js +105 -105
- package/bin/items.js +247 -247
- package/bin/logger.js +84 -84
- package/bin/meta.js +66 -66
- package/bin/naming.js +136 -112
- package/bin/util.js +61 -0
- package/bin/validate.js +39 -39
- package/package.json +62 -62
package/LICENSE
CHANGED
|
@@ -1,21 +1,21 @@
|
|
|
1
|
-
MIT License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2020 Joshua Pohl
|
|
4
|
-
|
|
5
|
-
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
-
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
-
in the Software without restriction, including without limitation the rights
|
|
8
|
-
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
-
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
-
furnished to do so, subject to the following conditions:
|
|
11
|
-
|
|
12
|
-
The above copyright notice and this permission notice shall be included in all
|
|
13
|
-
copies or substantial portions of the Software.
|
|
14
|
-
|
|
15
|
-
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
-
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
-
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
-
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
-
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
-
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
-
SOFTWARE.
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2020 Joshua Pohl
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,122 +1,146 @@
|
|
|
1
|
-
# podcast-dl
|
|
2
|
-
|
|
3
|
-
A humble CLI for downloading and archiving podcasts.
|
|
4
|
-
|
|
5
|
-
## How to Use
|
|
6
|
-
|
|
7
|
-
### npx
|
|
8
|
-
|
|
9
|
-
**[Node Required](https://nodejs.org/en/)**
|
|
10
|
-
|
|
11
|
-
`npx podcast-dl --url <PODCAST_RSS_URL>`
|
|
12
|
-
|
|
13
|
-
### Binaries
|
|
14
|
-
|
|
15
|
-
[Visit the releases page](https://github.com/lightpohl/podcast-dl/releases) and download the latest binary for your system.
|
|
16
|
-
|
|
17
|
-
`podcast-dl --url <PODCAST_RSS_URL>`
|
|
18
|
-
|
|
19
|
-
### [More Examples](./docs/examples.md)
|
|
20
|
-
|
|
21
|
-
## Options
|
|
22
|
-
|
|
23
|
-
Either `--url` or `--file` must be provided.
|
|
24
|
-
|
|
25
|
-
Type values surrounded in square brackets (`[]`) can be used as
|
|
26
|
-
|
|
27
|
-
| Option | Type | Required | Description
|
|
28
|
-
| --------------------------------- | ------------------- | -------- |
|
|
29
|
-
| --url | String | true\* | URL to podcast RSS feed.
|
|
30
|
-
| --file | String | true\* | Path to local RSS file.
|
|
31
|
-
| --out-dir | String | false | Specify output directory for episodes and metadata. Defaults to "./{{podcast_title}}"
|
|
32
|
-
| --threads | Number | false | Determines the number of downloads that will happen concurrently. Default is 1
|
|
33
|
-
| --attempts | Number | false | Sets the number of download attempts per individual file. Default is 3
|
|
34
|
-
| --archive | [String] | false | Download or write out items not listed in archive file. Generates archive file at path if not found. Defaults to "./{{podcast_title}}/archive.json" when used as a boolean option. See "Template Options" for more details. |
|
|
35
|
-
| --episode-template | String | false | Template for generating episode related filenames. See "Template Options" for details.
|
|
36
|
-
| --episode-custom-template-options | <String...> | false | Provide custom options for the episode template. See "Template Options" for details.
|
|
37
|
-
| --include-meta | | false | Write out podcast metadata to JSON.
|
|
38
|
-
| --include-episode-meta | | false | Write out individual episode metadata
|
|
39
|
-
| --include-episode-images | | false | Download found episode images.
|
|
40
|
-
| --include-episode-transcripts | | false | Download found episode transcripts.
|
|
41
|
-
| --offset | Number | false | Offset starting download position. Default is 0
|
|
42
|
-
| --limit | Number | false | Max number of episodes to download. Downloads all by default.
|
|
43
|
-
| --after | String | false | Only download episodes after this date (i.e. MM/DD/
|
|
44
|
-
| --before | String | false | Only download episodes before this date (i.e. MM/DD/
|
|
45
|
-
| --episode-regex | String | false | Match episode title against provided regex before starting download.
|
|
46
|
-
| --episode-regex-exclude | String | false |
|
|
47
|
-
| --episode-digits | Number | false | Minimum number of digits to use for episode numbering (e.g. 3 would generate "001" instead of "1"). Default is
|
|
48
|
-
| --episode-num-offset | Number | false | Offset the acquired episode number. Default is 0
|
|
49
|
-
| --episode-source-order | String | false | Attempted order to extract episode audio URL from RSS feed. Default is "enclosure,link"
|
|
50
|
-
| --episode-transcript-types | String | false | List of allowed transcript types in preferred order. Default is "application/json,application/x-subrip,application/srr,application/srt,text/vtt,text/html,text/plain".
|
|
51
|
-
| --season | Number | false | Only download episodes from
|
|
52
|
-
| --add-mp3-metadata | | false | Attempts to add a base level of episode metadata to each episode. Recommended only in cases where the original metadata is of poor quality. (**ffmpeg required**)
|
|
53
|
-
| --adjust-bitrate | String (e.g. "48k") | false | Attempts to adjust bitrate of episodes. (**ffmpeg required**)
|
|
54
|
-
| --mono | | false | Attempts to force episodes into mono. (**ffmpeg required**)
|
|
55
|
-
| --override | | false | Override local files on collision.
|
|
56
|
-
| --always-postprocess | | false | Always run additional tasks on the file regardless if the file already exists. This includes
|
|
57
|
-
| --reverse | | false | Reverse download direction and start at last RSS item.
|
|
58
|
-
| --info | | false | Print retrieved podcast info instead of downloading.
|
|
59
|
-
| --list | [String] | false | Print episode list instead of downloading. Defaults to "table" when used as a boolean option. "json" is also supported.
|
|
60
|
-
| --exec | String | false | Execute a command after each episode is downloaded. See "Template Options" for more details.
|
|
61
|
-
| --parser-config | String | false | Path to JSON file that will be parsed and used to override the default config passed to [rss-parser](https://github.com/rbren/rss-parser#xml-options).
|
|
62
|
-
| --user-agent | String | false | Specify custom user agent string for HTTP requests. Defaults to a Chrome user agent if not specified.
|
|
63
|
-
| --proxy | | false | Enable proxy support. Specify environment variables listed by [global-agent](https://github.com/gajus/global-agent#environment-variables).
|
|
64
|
-
| --help | | false | Output usage information.
|
|
65
|
-
|
|
66
|
-
## Archive
|
|
67
|
-
|
|
68
|
-
- If passed the `--archive [path]` option, `podcast-dl` will generate/use a JSON archive at the provided path.
|
|
69
|
-
- Before downloading an episode or writing out metadata, it'll check if the item was saved previously and abort the save if found.
|
|
70
|
-
|
|
71
|
-
## Template Options
|
|
72
|
-
|
|
73
|
-
Options that support templates allow users to specify a template for the generated filename(s) or option. The provided template will replace all matched keywords with the related data described below. Each keyword must be wrapped in two braces like so:
|
|
74
|
-
|
|
75
|
-
`--out-dir "./{{podcast_title}}"`
|
|
76
|
-
|
|
77
|
-
`--episode-template "{{release_date}}-{{title}}"`
|
|
78
|
-
|
|
79
|
-
### `--out-dir` & `--archive`
|
|
80
|
-
|
|
81
|
-
- `podcast_title`: Title of the podcast feed.
|
|
82
|
-
- `podcast_link`: `link` value provided for the podcast feed. Typically the homepage URL.
|
|
83
|
-
|
|
84
|
-
### `--episode-template`
|
|
85
|
-
|
|
86
|
-
- `title`: The title of the episode.
|
|
87
|
-
- `release_date`: The release date of the episode in `YYYYMMDD` format.
|
|
88
|
-
- `release_year`: The release year (`YYYY`) of the episode.
|
|
89
|
-
- `release_month`: The release month (`MM`) of the episode.
|
|
90
|
-
- `release_day`: The release day (`DD`) of the episode.
|
|
91
|
-
- `episode_num`: The
|
|
92
|
-
- `url`: URL of episode audio file.
|
|
93
|
-
- `duration`: Provided `mm:ss` duration (if found).
|
|
94
|
-
- `podcast_title`: Title of the podcast feed.
|
|
95
|
-
- `podcast_link`: `link` value provided for the podcast feed. Typically the homepage URL.
|
|
96
|
-
- `guid`: The GUID of the episode.
|
|
97
|
-
|
|
98
|
-
#### `--episode-custom-template-options`
|
|
99
|
-
|
|
100
|
-
Each matcher provided will be used to extract a value from the episode `title`. Access these values in the template using the `custom_<n>` keyword where `<n>` is the index of the matcher provided (starting from `0`).
|
|
101
|
-
|
|
102
|
-
If no match is found, the `custom_<n>` keyword will be replaced with an empty string.
|
|
103
|
-
|
|
104
|
-
### `--exec`
|
|
105
|
-
|
|
106
|
-
- `episode_path`: The path to the downloaded episode.
|
|
107
|
-
- `episode_path_base`: The path to the folder of the downloaded episode.
|
|
108
|
-
- `episode_filename`: The filename of the episode.
|
|
109
|
-
- `episode_filename_base`: The filename of the episode without its extension.
|
|
110
|
-
- `url`: URL of episode audio file.
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
- `
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
1
|
+
# podcast-dl
|
|
2
|
+
|
|
3
|
+
A humble CLI for downloading and archiving podcasts.
|
|
4
|
+
|
|
5
|
+
## How to Use
|
|
6
|
+
|
|
7
|
+
### npx
|
|
8
|
+
|
|
9
|
+
**[Node Required](https://nodejs.org/en/)**
|
|
10
|
+
|
|
11
|
+
`npx podcast-dl --url <PODCAST_RSS_URL>`
|
|
12
|
+
|
|
13
|
+
### Binaries
|
|
14
|
+
|
|
15
|
+
[Visit the releases page](https://github.com/lightpohl/podcast-dl/releases) and download the latest binary for your system.
|
|
16
|
+
|
|
17
|
+
`podcast-dl --url <PODCAST_RSS_URL>`
|
|
18
|
+
|
|
19
|
+
### [More Examples](./docs/examples.md)
|
|
20
|
+
|
|
21
|
+
## Options
|
|
22
|
+
|
|
23
|
+
Either `--url` or `--file` must be provided.
|
|
24
|
+
|
|
25
|
+
Type values surrounded in square brackets (`[]`) can be used as boolean options (no argument required).
|
|
26
|
+
|
|
27
|
+
| Option | Type | Required | Description |
|
|
28
|
+
| --------------------------------- | ------------------- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
|
|
29
|
+
| --url | String | true\* | URL to podcast RSS feed. |
|
|
30
|
+
| --file | String | true\* | Path to local RSS file. |
|
|
31
|
+
| --out-dir | String | false | Specify output directory for episodes and metadata. Defaults to `"./{{podcast_title}}"`. See "Template Options" for more details. |
|
|
32
|
+
| --threads | Number | false | Determines the number of downloads that will happen concurrently. Default is `1`. |
|
|
33
|
+
| --attempts | Number | false | Sets the number of download attempts per individual file. Default is `3`. |
|
|
34
|
+
| --archive | [String] | false | Download or write out items not listed in archive file. Generates archive file at path if not found. Defaults to `"./{{podcast_title}}/archive.json"` when used as a boolean option. See "Template Options" for more details. |
|
|
35
|
+
| --episode-template | String | false | Template for generating episode related filenames. See "Template Options" for details. |
|
|
36
|
+
| --episode-custom-template-options | <String...> | false | Provide custom options for the episode template. See "Template Options" for details. |
|
|
37
|
+
| --include-meta | | false | Write out podcast metadata to JSON. |
|
|
38
|
+
| --include-episode-meta | | false | Write out individual episode metadata to JSON. |
|
|
39
|
+
| --include-episode-images | | false | Download found episode images. |
|
|
40
|
+
| --include-episode-transcripts | | false | Download found episode transcripts. |
|
|
41
|
+
| --offset | Number | false | Offset starting download position. Default is `0`. |
|
|
42
|
+
| --limit | Number | false | Max number of episodes to download. Downloads all by default. |
|
|
43
|
+
| --after | String | false | Only download episodes after this date (i.e. MM/DD/YYYY, inclusive). |
|
|
44
|
+
| --before | String | false | Only download episodes before this date (i.e. MM/DD/YYYY, inclusive). |
|
|
45
|
+
| --episode-regex | String | false | Match episode title against provided regex before starting download. |
|
|
46
|
+
| --episode-regex-exclude | String | false | Episode titles matching provided regex will be excluded. |
|
|
47
|
+
| --episode-digits | Number | false | Minimum number of digits to use for episode numbering (e.g. 3 would generate "001" instead of "1"). Default is `1`. |
|
|
48
|
+
| --episode-num-offset | Number | false | Offset the acquired episode number. Default is `0`. |
|
|
49
|
+
| --episode-source-order | String | false | Attempted order to extract episode audio URL from RSS feed. Default is `"enclosure,link"`. |
|
|
50
|
+
| --episode-transcript-types | String | false | List of allowed transcript types in preferred order. Default is "application/json,application/x-subrip,application/srr,application/srt,text/vtt,text/html,text/plain". |
|
|
51
|
+
| --season | Number | false | Only download episodes from specified season. Note: this will only work if the RSS feed includes the `itunes:season` tag on episodes. |
|
|
52
|
+
| --add-mp3-metadata | | false | Attempts to add a base level of episode metadata to each episode. Recommended only in cases where the original metadata is of poor quality. (**ffmpeg required**) |
|
|
53
|
+
| --adjust-bitrate | String (e.g. "48k") | false | Attempts to adjust bitrate of episodes. (**ffmpeg required**) |
|
|
54
|
+
| --mono | | false | Attempts to force episodes into mono. (**ffmpeg required**) |
|
|
55
|
+
| --override | | false | Override local files on collision. |
|
|
56
|
+
| --always-postprocess | | false | Always run additional tasks on the file regardless if the file already exists. This includes `--add-mp3-metadata`, `--adjust-bitrate`, `--mono`, and `--exec`. |
|
|
57
|
+
| --reverse | | false | Reverse download direction and start at last RSS item. |
|
|
58
|
+
| --info | | false | Print retrieved podcast info instead of downloading. |
|
|
59
|
+
| --list | [String] | false | Print episode list instead of downloading. Defaults to `"table"` when used as a boolean option. `"json"` is also supported. |
|
|
60
|
+
| --exec | String | false | Execute a command after each episode is downloaded. See "Template Options" for more details. |
|
|
61
|
+
| --parser-config | String | false | Path to JSON file that will be parsed and used to override the default config passed to [rss-parser](https://github.com/rbren/rss-parser#xml-options). |
|
|
62
|
+
| --user-agent | String | false | Specify custom user agent string for HTTP requests. Defaults to a Chrome user agent if not specified. |
|
|
63
|
+
| --proxy | | false | Enable proxy support. Specify environment variables listed by [global-agent](https://github.com/gajus/global-agent#environment-variables). |
|
|
64
|
+
| --help | | false | Output usage information. |
|
|
65
|
+
|
|
66
|
+
## Archive
|
|
67
|
+
|
|
68
|
+
- If passed the `--archive [path]` option, `podcast-dl` will generate/use a JSON archive at the provided path.
|
|
69
|
+
- Before downloading an episode or writing out metadata, it'll check if the item was saved previously and abort the save if found.
|
|
70
|
+
|
|
71
|
+
## Template Options
|
|
72
|
+
|
|
73
|
+
Options that support templates allow users to specify a template for the generated filename(s) or option. The provided template will replace all matched keywords with the related data described below. Each keyword must be wrapped in two braces like so:
|
|
74
|
+
|
|
75
|
+
`--out-dir "./{{podcast_title}}"`
|
|
76
|
+
|
|
77
|
+
`--episode-template "{{release_date}}-{{title}}"`
|
|
78
|
+
|
|
79
|
+
### `--out-dir` & `--archive`
|
|
80
|
+
|
|
81
|
+
- `podcast_title`: Title of the podcast feed.
|
|
82
|
+
- `podcast_link`: `link` value provided for the podcast feed. Typically the homepage URL.
|
|
83
|
+
|
|
84
|
+
### `--episode-template`
|
|
85
|
+
|
|
86
|
+
- `title`: The title of the episode.
|
|
87
|
+
- `release_date`: The release date of the episode in `YYYYMMDD` format.
|
|
88
|
+
- `release_year`: The release year (`YYYY`) of the episode.
|
|
89
|
+
- `release_month`: The release month (`MM`) of the episode.
|
|
90
|
+
- `release_day`: The release day (`DD`) of the episode.
|
|
91
|
+
- `episode_num`: The position number of where the episode appears in the feed.
|
|
92
|
+
- `url`: URL of episode audio file.
|
|
93
|
+
- `duration`: Provided `mm:ss` duration (if found).
|
|
94
|
+
- `podcast_title`: Title of the podcast feed.
|
|
95
|
+
- `podcast_link`: `link` value provided for the podcast feed. Typically the homepage URL.
|
|
96
|
+
- `guid`: The GUID of the episode.
|
|
97
|
+
|
|
98
|
+
#### `--episode-custom-template-options`
|
|
99
|
+
|
|
100
|
+
Each matcher provided will be used to extract a value from the episode `title`. Access these values in the template using the `custom_<n>` keyword where `<n>` is the index of the matcher provided (starting from `0`).
|
|
101
|
+
|
|
102
|
+
If no match is found, the `custom_<n>` keyword will be replaced with an empty string.
|
|
103
|
+
|
|
104
|
+
### `--exec`
|
|
105
|
+
|
|
106
|
+
- `episode_path`: The path to the downloaded episode.
|
|
107
|
+
- `episode_path_base`: The path to the folder of the downloaded episode.
|
|
108
|
+
- `episode_filename`: The filename of the episode.
|
|
109
|
+
- `episode_filename_base`: The filename of the episode without its extension.
|
|
110
|
+
- `url`: URL of episode audio file.
|
|
111
|
+
|
|
112
|
+
### Template Filters
|
|
113
|
+
|
|
114
|
+
Template variables can be transformed using filters. Filters are applied using the pipe (`|`) character and can be chained:
|
|
115
|
+
|
|
116
|
+
`--episode-template "{{podcast_title|underscore}}-{{title|strip_special|camelcase}}"`
|
|
117
|
+
|
|
118
|
+
For example, given `title` = "Serial- S01 E01: The Alibi":
|
|
119
|
+
|
|
120
|
+
- `{{title|strip_special|underscore}}` produces `Serial S01 E01 The Alibi` then `Serial_S01_E01_The_Alibi`
|
|
121
|
+
- `{{title|strip_special|camelcase}}` produces `SerialS01E01TheAlibi`
|
|
122
|
+
|
|
123
|
+
#### Available Filters
|
|
124
|
+
|
|
125
|
+
| Filter | Description | Input | Output |
|
|
126
|
+
| --------------- | --------------------------------------------- | ------------- | ----------- |
|
|
127
|
+
| `strip` | Remove all whitespace | `"foo bar"` | `"foobar"` |
|
|
128
|
+
| `strip_special` | Remove non-alphanumeric chars (except spaces) | `"S01: E01!"` | `"S01 E01"` |
|
|
129
|
+
| `underscore` | Replace whitespace with underscores | `"foo bar"` | `"foo_bar"` |
|
|
130
|
+
| `dash` | Replace whitespace with dashes | `"foo bar"` | `"foo-bar"` |
|
|
131
|
+
| `camelcase` | Convert to UpperCamelCase | `"foo bar"` | `"FooBar"` |
|
|
132
|
+
| `lowercase` | Convert to lowercase | `"FOO Bar"` | `"foo bar"` |
|
|
133
|
+
| `uppercase` | Convert to UPPERCASE | `"foo bar"` | `"FOO BAR"` |
|
|
134
|
+
| `trim` | Remove leading/trailing whitespace | `" foo "` | `"foo"` |
|
|
135
|
+
|
|
136
|
+
## Log Levels
|
|
137
|
+
|
|
138
|
+
By default, all logs and errors are outputted to the console. The amount of logs can be controlled using the environment variable `LOG_LEVEL` with the following options:
|
|
139
|
+
|
|
140
|
+
- `static`: All logs and errors are outputted to the console, but disables any animations.
|
|
141
|
+
- `quiet`: Only important info and non-critical errors will be logged (e.g. episode download started).
|
|
142
|
+
- `silent`: Only critical error messages will be logged.
|
|
143
|
+
|
|
144
|
+
## OS Filename Limits
|
|
145
|
+
|
|
146
|
+
By default, the max length of a generated filename is `255`. If your OS has different limitations, or if you're running into issues with non-standard feeds, you can adjust the limit via the environment variable `MAX_LENGTH_FILENAME`.
|
package/bin/archive.js
CHANGED
|
@@ -1,39 +1,39 @@
|
|
|
1
|
-
import dayjs from "dayjs";
|
|
2
|
-
import fs from "fs";
|
|
3
|
-
import path from "path";
|
|
4
|
-
import { getJsonFile } from "./util.js";
|
|
5
|
-
|
|
6
|
-
export const getArchiveKey = ({ prefix, name }) => {
|
|
7
|
-
return `${prefix}-${name}`;
|
|
8
|
-
};
|
|
9
|
-
|
|
10
|
-
export const getArchive = (archive) => {
|
|
11
|
-
const archiveContent = getJsonFile(archive);
|
|
12
|
-
return archiveContent === null ? [] : archiveContent;
|
|
13
|
-
};
|
|
14
|
-
|
|
15
|
-
export const writeToArchive = ({ key, archive }) => {
|
|
16
|
-
const archivePath = path.resolve(process.cwd(), archive);
|
|
17
|
-
const archiveResult = getArchive(archive);
|
|
18
|
-
|
|
19
|
-
if (!archiveResult.includes(key)) {
|
|
20
|
-
archiveResult.push(key);
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
fs.writeFileSync(archivePath, JSON.stringify(archiveResult, null, 4));
|
|
24
|
-
};
|
|
25
|
-
|
|
26
|
-
export const getIsInArchive = ({ key, archive }) => {
|
|
27
|
-
const archiveResult = getArchive(archive);
|
|
28
|
-
return archiveResult.includes(key);
|
|
29
|
-
};
|
|
30
|
-
|
|
31
|
-
export const getArchiveFilename = ({ pubDate, name, ext }) => {
|
|
32
|
-
const formattedPubDate = pubDate
|
|
33
|
-
? dayjs(new Date(pubDate)).format("YYYYMMDD")
|
|
34
|
-
: null;
|
|
35
|
-
|
|
36
|
-
const baseName = formattedPubDate ? `${formattedPubDate}-${name}` : name;
|
|
37
|
-
|
|
38
|
-
return `${baseName}${ext}`;
|
|
39
|
-
};
|
|
1
|
+
import dayjs from "dayjs";
|
|
2
|
+
import fs from "fs";
|
|
3
|
+
import path from "path";
|
|
4
|
+
import { getJsonFile } from "./util.js";
|
|
5
|
+
|
|
6
|
+
export const getArchiveKey = ({ prefix, name }) => {
|
|
7
|
+
return `${prefix}-${name}`;
|
|
8
|
+
};
|
|
9
|
+
|
|
10
|
+
export const getArchive = (archive) => {
|
|
11
|
+
const archiveContent = getJsonFile(archive);
|
|
12
|
+
return archiveContent === null ? [] : archiveContent;
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
export const writeToArchive = ({ key, archive }) => {
|
|
16
|
+
const archivePath = path.resolve(process.cwd(), archive);
|
|
17
|
+
const archiveResult = getArchive(archive);
|
|
18
|
+
|
|
19
|
+
if (!archiveResult.includes(key)) {
|
|
20
|
+
archiveResult.push(key);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
fs.writeFileSync(archivePath, JSON.stringify(archiveResult, null, 4));
|
|
24
|
+
};
|
|
25
|
+
|
|
26
|
+
export const getIsInArchive = ({ key, archive }) => {
|
|
27
|
+
const archiveResult = getArchive(archive);
|
|
28
|
+
return archiveResult.includes(key);
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
export const getArchiveFilename = ({ pubDate, name, ext }) => {
|
|
32
|
+
const formattedPubDate = pubDate
|
|
33
|
+
? dayjs(new Date(pubDate)).format("YYYYMMDD")
|
|
34
|
+
: null;
|
|
35
|
+
|
|
36
|
+
const baseName = formattedPubDate ? `${formattedPubDate}-${name}` : name;
|
|
37
|
+
|
|
38
|
+
return `${baseName}${ext}`;
|
|
39
|
+
};
|
package/bin/async.js
CHANGED
|
@@ -22,6 +22,7 @@ import {
|
|
|
22
22
|
import { writeItemMeta } from "./meta.js";
|
|
23
23
|
import { getItemFilename } from "./naming.js";
|
|
24
24
|
import {
|
|
25
|
+
correctExtensionFromMime,
|
|
25
26
|
getEpisodeAudioUrlAndExt,
|
|
26
27
|
getTempPath,
|
|
27
28
|
prepareOutputPath,
|
|
@@ -56,12 +57,12 @@ export const download = async (options) => {
|
|
|
56
57
|
await onAfterDownload();
|
|
57
58
|
}
|
|
58
59
|
|
|
59
|
-
return;
|
|
60
|
+
return outputPath;
|
|
60
61
|
}
|
|
61
62
|
|
|
62
63
|
if (key && archive && getIsInArchive({ key, archive })) {
|
|
63
64
|
logMessage("Download exists in archive. Skipping...");
|
|
64
|
-
return;
|
|
65
|
+
return null;
|
|
65
66
|
}
|
|
66
67
|
|
|
67
68
|
let headResponse = null;
|
|
@@ -125,7 +126,7 @@ export const download = async (options) => {
|
|
|
125
126
|
if (attempt <= maxAttempts) {
|
|
126
127
|
logMessage(`Download attempt #${attempt} failed. Retrying...`);
|
|
127
128
|
|
|
128
|
-
await download({
|
|
129
|
+
return await download({
|
|
129
130
|
...options,
|
|
130
131
|
attempt: attempt + 1,
|
|
131
132
|
});
|
|
@@ -144,24 +145,38 @@ export const download = async (options) => {
|
|
|
144
145
|
LOG_LEVELS.important
|
|
145
146
|
);
|
|
146
147
|
|
|
147
|
-
return;
|
|
148
|
+
return null;
|
|
148
149
|
}
|
|
149
150
|
|
|
150
|
-
|
|
151
|
+
const { outputPath: finalOutputPath, key: finalKey } =
|
|
152
|
+
correctExtensionFromMime({
|
|
153
|
+
outputPath,
|
|
154
|
+
key,
|
|
155
|
+
contentType: headResponse?.headers?.["content-type"],
|
|
156
|
+
onCorrect: (from, to) =>
|
|
157
|
+
logMessage(
|
|
158
|
+
`Correcting extension: ${from} --> ${to}`,
|
|
159
|
+
LOG_LEVELS.important
|
|
160
|
+
),
|
|
161
|
+
});
|
|
162
|
+
|
|
163
|
+
fs.renameSync(tempOutputPath, finalOutputPath);
|
|
151
164
|
|
|
152
165
|
logMessage("Download complete!");
|
|
153
166
|
|
|
154
167
|
if (onAfterDownload) {
|
|
155
|
-
await onAfterDownload();
|
|
168
|
+
await onAfterDownload(finalOutputPath);
|
|
156
169
|
}
|
|
157
170
|
|
|
158
|
-
if (
|
|
171
|
+
if (finalKey && archive) {
|
|
159
172
|
try {
|
|
160
|
-
writeToArchive({ key, archive });
|
|
173
|
+
writeToArchive({ key: finalKey, archive });
|
|
161
174
|
} catch (error) {
|
|
162
175
|
throw new Error(`Error writing to archive: ${error.toString()}`);
|
|
163
176
|
}
|
|
164
177
|
}
|
|
178
|
+
|
|
179
|
+
return finalOutputPath;
|
|
165
180
|
};
|
|
166
181
|
|
|
167
182
|
export const downloadItemsAsync = async ({
|
|
@@ -236,10 +251,10 @@ export const downloadItemsAsync = async ({
|
|
|
236
251
|
maxAttempts: attempts,
|
|
237
252
|
outputPath: outputPodcastPath,
|
|
238
253
|
url: episodeAudioUrl,
|
|
239
|
-
onAfterDownload: async () => {
|
|
254
|
+
onAfterDownload: async (finalEpisodePath) => {
|
|
240
255
|
if (item._episodeImage) {
|
|
241
256
|
try {
|
|
242
|
-
await download({
|
|
257
|
+
const finalImagePath = await download({
|
|
243
258
|
archive,
|
|
244
259
|
override,
|
|
245
260
|
userAgent,
|
|
@@ -249,6 +264,10 @@ export const downloadItemsAsync = async ({
|
|
|
249
264
|
outputPath: item._episodeImage.outputPath,
|
|
250
265
|
url: item._episodeImage.url,
|
|
251
266
|
});
|
|
267
|
+
|
|
268
|
+
if (finalImagePath) {
|
|
269
|
+
item._episodeImage.outputPath = finalImagePath;
|
|
270
|
+
}
|
|
252
271
|
} catch (error) {
|
|
253
272
|
hasErrors = true;
|
|
254
273
|
logError(
|
|
@@ -261,7 +280,7 @@ export const downloadItemsAsync = async ({
|
|
|
261
280
|
|
|
262
281
|
if (item._episodeTranscript) {
|
|
263
282
|
try {
|
|
264
|
-
await download({
|
|
283
|
+
const finalTranscriptPath = await download({
|
|
265
284
|
archive,
|
|
266
285
|
override,
|
|
267
286
|
key: item._episodeTranscript.key,
|
|
@@ -271,6 +290,10 @@ export const downloadItemsAsync = async ({
|
|
|
271
290
|
url: item._episodeTranscript.url,
|
|
272
291
|
userAgent,
|
|
273
292
|
});
|
|
293
|
+
|
|
294
|
+
if (finalTranscriptPath) {
|
|
295
|
+
item._episodeTranscript.outputPath = finalTranscriptPath;
|
|
296
|
+
}
|
|
274
297
|
} catch (error) {
|
|
275
298
|
hasErrors = true;
|
|
276
299
|
logError(
|
|
@@ -292,7 +315,7 @@ export const downloadItemsAsync = async ({
|
|
|
292
315
|
bitrate,
|
|
293
316
|
mono,
|
|
294
317
|
itemIndex: item._originalIndex,
|
|
295
|
-
outputPath:
|
|
318
|
+
outputPath: finalEpisodePath,
|
|
296
319
|
episodeImageOutputPath: hasEpisodeImage
|
|
297
320
|
? item._episodeImage.outputPath
|
|
298
321
|
: undefined,
|
|
@@ -310,7 +333,7 @@ export const downloadItemsAsync = async ({
|
|
|
310
333
|
await runExec({
|
|
311
334
|
exec,
|
|
312
335
|
basePath,
|
|
313
|
-
outputPodcastPath,
|
|
336
|
+
outputPodcastPath: finalEpisodePath,
|
|
314
337
|
episodeFilename,
|
|
315
338
|
episodeAudioUrl,
|
|
316
339
|
});
|
package/bin/bin.js
CHANGED
|
@@ -148,10 +148,7 @@ const main = async () => {
|
|
|
148
148
|
const podcastImageFileExt = getUrlExt(podcastImageUrl);
|
|
149
149
|
const outputImagePath = _path.resolve(
|
|
150
150
|
basePath,
|
|
151
|
-
getSimpleFilename(
|
|
152
|
-
feed.title ? feed.title : "image",
|
|
153
|
-
feed.title ? `.image${podcastImageFileExt}` : podcastImageFileExt
|
|
154
|
-
)
|
|
151
|
+
getSimpleFilename(feed.title || "image", podcastImageFileExt)
|
|
155
152
|
);
|
|
156
153
|
|
|
157
154
|
try {
|
|
@@ -163,9 +160,7 @@ const main = async () => {
|
|
|
163
160
|
marker: podcastImageUrl,
|
|
164
161
|
key: getArchiveKey({
|
|
165
162
|
prefix: archivePrefix,
|
|
166
|
-
name: `${
|
|
167
|
-
feed.title ? `${feed.title}.image` : "image"
|
|
168
|
-
}${podcastImageFileExt}`,
|
|
163
|
+
name: `${feed.title || "image"}${podcastImageFileExt}`,
|
|
169
164
|
}),
|
|
170
165
|
outputPath: outputImagePath,
|
|
171
166
|
url: podcastImageUrl,
|