First upload version 0.0.1

This commit is contained in:
Neyra
2026-02-05 15:27:49 +08:00
commit 8e9b7201ed
4182 changed files with 593136 additions and 0 deletions

21
node_modules/ipull/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2023 Ido S.
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

345
node_modules/ipull/README.md generated vendored Normal file
View File

@@ -0,0 +1,345 @@
<div align="center">
<h1>iPull</h1>
<img src="./assets/ipull-logo-rounded.png" height="200px" />
</div>
<div align="center">
[![Build](https://github.com/ido-pluto/ipull/actions/workflows/build.yml/badge.svg)](https://github.com/ido-pluto/ipull/actions/workflows/build.yml)
[![License](https://badgen.net/badge/color/MIT/green?label=license)](https://www.npmjs.com/package/ipull)
[![Types](https://badgen.net/badge/color/TypeScript/blue?label=types)](https://www.npmjs.com/package/ipull)
[![npm downloads](https://img.shields.io/npm/dt/ipull.svg)](https://www.npmjs.com/package/ipull)
[![Version](https://badgen.net/npm/v/ipull)](https://www.npmjs.com/package/ipull)
</div>
<br />
> Super fast file downloader with multiple connections
```bash
npx ipull http://example.com/file.large
```
![pull-example](./assets/pull-file.gif)
## Features
- Download using parallels connections
- Pausing and resuming downloads
- Node.js and browser support
- Smart retry on fail
- CLI Progress bar
- Download statistics (speed, time left, etc.)
### NodeJS API
```ts
import {downloadFile} from 'ipull';
const downloader = await downloadFile({
url: 'https://example.com/file.large',
directory: './this/path', // or 'savePath' for full path
cliProgress: true, // Show progress bar in the CLI (default: false)
parallelStreams: 3 // Number of parallel connections (default: 3)
});
await downloader.download();
```
## Browser support
Download a file in the browser using multiple connections
```ts
import {downloadFileBrowser} from "ipull/dist/browser.js";
const downloader = await downloadFileBrowser({
url: 'https://example.com/file.large',
acceptRangeIsKnown: true // cors origin request will not return the range header, but we can force it to be true (multi-connection download)
});
await downloader.download();
image.src = downloader.writeStream.resultAsBlobURL();
console.log(downloader.writeStream.result); // Uint8Array
```
### Custom stream
You can use a custom stream
```ts
import {downloadFileBrowser} from "ipull/dist/browser.js";
const downloader = await downloadFileBrowser({
url: 'https://example.com/file.large',
onWrite: (cursor: number, buffer: Uint8Array, options) => {
console.log(`Writing ${buffer.length} bytes at cursor ${cursor}, with options: ${JSON.stringify(options)}`);
}
});
await downloader.download();
console.log(downloader.writeStream.result.length === 0); // true, because we write to a custom stream
```
## CLI
```
Usage: ipull [options] [files...]
Pull/copy files from a remote server/local directory
Arguments:
files Files to pull/copy
Options:
-s --save [path] Save location (directory/file)
-c --connections [number] Number of parallel connections (default: "4")
-p --program [type] The download strategy (choices: "stream", "chunks")
-t --truncate-name Truncate file names in the CLI status to make them appear shorter
-V, --version output the version number
-h, --help display help for command
Commands:
set [options] [path] <value> Set download locations
```
### Set custom save directory
You can set a custom save directory by using the `set` command.
```bash
ipull set .zip ~/Downloads/zips
```
(use `default` to set the default save directory)
## Advanced usage
### Skip existing files
Skip downloading files that already exist in the save location and have the same size.
```ts
import {downloadFile} from 'ipull';
const downloader = await downloadFile({
url: 'https://example.com/file.large',
directory: './this/path',
skipExisting: true
});
```
### Download file from parts
Consolidate multiple files parts into one file.
Beneficial for downloading large files from servers that limit file size. (e.g. HuggingFace models)
```ts
import {downloadFile} from 'ipull';
const downloadParts = [
"https://example.com/file.large-part-1",
"https://example.com/file.large-part-2",
"https://example.com/file.large-part-3",
];
const downloader = await downloadFile({
partURLs: downloadParts,
directory: './this/path',
filename: 'file.large'
});
await downloader.download();
```
** The split must be binary and not a zip-split
### Custom headers
You can set custom headers for the download request
```ts
import {downloadFile} from 'ipull';
const downloader = await downloadFile({
url: 'https://example.com/file.large',
savePath: './this/path/file.large',
headers: {
'Authorization': 'Bearer token 1'
},
// You can also add alternative headers in case of an 400-499 error
tryHeaders: [
{
Authorization: 'Bearer token 2'
}
]
});
await downloader.download();
```
### Abort download
You can cancel the download by calling the `close` method (it will not delete the file).
If you want to also delete the file, you can call the `closeAndDeleteFile` method.
```ts
import {downloadFile} from 'ipull';
const downloader = await downloadFile({
url: 'https://example.com/file.large',
directory: './this/path'
});
setTimeout(() => {
downloader.close();
}, 5_000);
await downloader.download();
```
### Pause & Resume download
```ts
import {downloadFile} from 'ipull';
const downloader = await downloadFile({
url: 'https://example.com/file.large',
directory: './this/path'
});
setInterval(() => {
downloader.pause();
setTimeout(() => {
downloader.resume();
}, 5_000);
}, 10_000);
await downloader.download();
```
** The pause may take a few seconds to actually pause the download, because it waits for the current connections to
finish
### Error handling
If a network/file-system error occurs, the download will automatically retry
with [async-retry](https://www.npmjs.com/package/async-retry)
If the maximum reties was reached the download will fail and an error will be thrown from the `download()` call:
```ts
import {downloadFile} from 'ipull';
const downloader = await downloadFile({
url: 'https://example.com/file.large',
directory: './this/path'
});
try {
await downloader.download();
} catch (error) {
console.error(`Download failed: ${error.message}`);
}
```
### Download Stuck
In some edge cases, the re-try mechanism may give the illusion that the download is stuck.
To debug this, disable the re-try mechanism:
```js
const downloader = await downloadFile({
url: 'https://example.com/file.large',
directory: './this/path',
retry: {
retries: 0
}
});
```
### Listening to events
Events are emitted using the `EventEmitter` pattern and can be listened to using the `on` method
```ts
interface DownloadEngineEvents {
start: [];
paused: [];
resumed: [];
progress: [FormattedStatus];
save: [DownloadProgressInfo];
finished: [];
closed: [];
}
const downloader = await downloadFile({
url: 'https://example.com/file.large',
directory: './this/path'
});
downloader.on("progress", (progress) => {
console.log(`Downloaded ${progress.transferred} bytes`);
});
```
### Download multiple files
If you want to download multiple files, you can use the `downloadSequence` function.
By default, it will download files one by one, but you can set the `parallel` option to download them in parallel.
It is better to download one file at a time if you are downloading from the same server (as it may limit the number of
connections).
```ts
import {downloadFile, downloadSequence} from "ipull";
const downloader = await downloadSequence(
{
cliProgress: true,
// parallelDownloads: 2, download 2 files in parallel, default is 1
},
downloadFile({
url: "https://example.com/file1.large",
directory: "."
}),
downloadFile({
url: "https://example.com/file2.large",
directory: "."
}),
);
console.log(`Downloading ${downloader.downloads.length} files...`);
await downloader.download();
```
### Custom progress bar
```ts
import {downloadFile, FormattedStatus} from "ipull";
function createProgressBar({fileName, ...data}: FormattedStatus) {
return `${fileName} ${JSON.stringify(data)}`;
}
const downloader = await downloadFile({
url: "https://example.com/file.large",
directory: "./this/path",
cliStyle: createProgressBar
});
await downloader.download();
```
<br />
<div align="center" width="360">
<img alt="Star please" src="./assets/star-please.png" width="360" margin="auto" />
<br/>
<p align="right">
<i>If you like this repo, star it ✨</i>&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;
</p>
</div>

19
node_modules/ipull/dist/browser.d.ts generated vendored Normal file
View File

@@ -0,0 +1,19 @@
import { downloadFileBrowser, DownloadFileBrowserOptions, downloadSequenceBrowser } from "./download/browser-download.js";
import DownloadEngineBrowser from "./download/download-engine/engine/download-engine-browser.js";
import EmptyResponseError from "./download/download-engine/streams/download-engine-fetch-stream/errors/empty-response-error.js";
import StatusCodeError from "./download/download-engine/streams/download-engine-fetch-stream/errors/status-code-error.js";
import XhrError from "./download/download-engine/streams/download-engine-fetch-stream/errors/xhr-error.js";
import { SaveProgressInfo } from "./download/download-engine/types.js";
import InvalidContentLengthError from "./download/download-engine/streams/download-engine-fetch-stream/errors/invalid-content-length-error.js";
import FetchStreamError from "./download/download-engine/streams/download-engine-fetch-stream/errors/fetch-stream-error.js";
import IpullError from "./errors/ipull-error.js";
import EngineError from "./download/download-engine/engine/error/engine-error.js";
import { FormattedStatus } from "./download/transfer-visualize/format-transfer-status.js";
import DownloadEngineMultiDownload from "./download/download-engine/engine/download-engine-multi-download.js";
import HttpError from "./download/download-engine/streams/download-engine-fetch-stream/errors/http-error.js";
import BaseDownloadEngine from "./download/download-engine/engine/base-download-engine.js";
import { InvalidOptionError } from "./download/download-engine/engine/error/InvalidOptionError.js";
import { DownloadFlags, DownloadStatus } from "./download/download-engine/download-file/progress-status-file.js";
import { NoDownloadEngineProvidedError } from "./download/download-engine/engine/error/no-download-engine-provided-error.js";
export { DownloadFlags, DownloadStatus, downloadFileBrowser, downloadSequenceBrowser, EmptyResponseError, HttpError, StatusCodeError, XhrError, InvalidContentLengthError, FetchStreamError, IpullError, EngineError, InvalidOptionError, NoDownloadEngineProvidedError };
export type { BaseDownloadEngine, DownloadFileBrowserOptions, DownloadEngineBrowser, DownloadEngineMultiDownload, FormattedStatus, SaveProgressInfo };

14
node_modules/ipull/dist/browser.js generated vendored Normal file
View File

@@ -0,0 +1,14 @@
import { downloadFileBrowser, downloadSequenceBrowser } from "./download/browser-download.js";
import EmptyResponseError from "./download/download-engine/streams/download-engine-fetch-stream/errors/empty-response-error.js";
import StatusCodeError from "./download/download-engine/streams/download-engine-fetch-stream/errors/status-code-error.js";
import XhrError from "./download/download-engine/streams/download-engine-fetch-stream/errors/xhr-error.js";
import InvalidContentLengthError from "./download/download-engine/streams/download-engine-fetch-stream/errors/invalid-content-length-error.js";
import FetchStreamError from "./download/download-engine/streams/download-engine-fetch-stream/errors/fetch-stream-error.js";
import IpullError from "./errors/ipull-error.js";
import EngineError from "./download/download-engine/engine/error/engine-error.js";
import HttpError from "./download/download-engine/streams/download-engine-fetch-stream/errors/http-error.js";
import { InvalidOptionError } from "./download/download-engine/engine/error/InvalidOptionError.js";
import { DownloadFlags, DownloadStatus } from "./download/download-engine/download-file/progress-status-file.js";
import { NoDownloadEngineProvidedError } from "./download/download-engine/engine/error/no-download-engine-provided-error.js";
export { DownloadFlags, DownloadStatus, downloadFileBrowser, downloadSequenceBrowser, EmptyResponseError, HttpError, StatusCodeError, XhrError, InvalidContentLengthError, FetchStreamError, IpullError, EngineError, InvalidOptionError, NoDownloadEngineProvidedError };
//# sourceMappingURL=browser.js.map

1
node_modules/ipull/dist/browser.js.map generated vendored Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"browser.js","sourceRoot":"","sources":["../src/browser.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,mBAAmB,EAA8B,uBAAuB,EAAC,MAAM,gCAAgC,CAAC;AAExH,OAAO,kBAAkB,MAAM,gGAAgG,CAAC;AAChI,OAAO,eAAe,MAAM,6FAA6F,CAAC;AAC1H,OAAO,QAAQ,MAAM,qFAAqF,CAAC;AAE3G,OAAO,yBAAyB,MAAM,wGAAwG,CAAC;AAC/I,OAAO,gBAAgB,MAAM,8FAA8F,CAAC;AAC5H,OAAO,UAAU,MAAM,yBAAyB,CAAC;AACjD,OAAO,WAAW,MAAM,yDAAyD,CAAC;AAGlF,OAAO,SAAS,MAAM,sFAAsF,CAAC;AAE7G,OAAO,EAAC,kBAAkB,EAAC,MAAM,+DAA+D,CAAC;AACjG,OAAO,EAAC,aAAa,EAAE,cAAc,EAAC,MAAM,kEAAkE,CAAC;AAC/G,OAAO,EAAC,6BAA6B,EAAC,MAAM,8EAA8E,CAAC;AAE3H,OAAO,EACH,aAAa,EACb,cAAc,EACd,mBAAmB,EACnB,uBAAuB,EACvB,kBAAkB,EAClB,SAAS,EACT,eAAe,EACf,QAAQ,EACR,yBAAyB,EACzB,gBAAgB,EAChB,UAAU,EACV,WAAW,EACX,kBAAkB,EAClB,6BAA6B,EAChC,CAAC"}

2
node_modules/ipull/dist/cli/cli.d.ts generated vendored Normal file
View File

@@ -0,0 +1,2 @@
#!/usr/bin/env node
export {};

45
node_modules/ipull/dist/cli/cli.js generated vendored Normal file
View File

@@ -0,0 +1,45 @@
#!/usr/bin/env node
import path from "path";
import { Command, Option } from "commander";
import { packageJson } from "../const.js";
import { downloadFile, downloadSequence } from "../download/node-download.js";
import { setCommand } from "./commands/set.js";
import findDownloadDir, { downloadToDirectory, findFileName } from "./utils/find-download-dir.js";
const pullCommand = new Command();
pullCommand
.description("Pull/copy files from remote server/local directory")
.argument("[files...]", "Files to pull/copy")
.option("-s --save [path]", "Save location (directory/file)")
.option("-c --connections [number]", "Number of parallel connections", "4")
.addOption(new Option("-st --style [type]", "The style of the CLI progress bar").choices(["basic", "fancy", "ci", "summary"]))
.addOption(new Option("-p --program [type]", "The download strategy").choices(["stream", "chunks"]))
.option("-t --truncate-name", "Truncate file names in the CLI status to make them appear shorter")
.action(async (files = [], { save: saveLocation, truncateName, number, program, style }) => {
if (files.length === 0) {
pullCommand.outputHelp();
process.exit(0);
}
const fileDownloads = await Promise.all(files.map(async (file) => {
const isDirectory = saveLocation && await downloadToDirectory(saveLocation);
const directory = isDirectory ? saveLocation : await findDownloadDir(findFileName(file));
const fileName = isDirectory || !saveLocation ? "" : path.basename(saveLocation);
return await downloadFile({
url: file,
directory,
fileName,
truncateName,
parallelStreams: Number(number) || 4,
programType: program
});
}));
const downloader = await downloadSequence({
truncateName,
cliProgress: true,
cliStyle: style
}, ...fileDownloads);
await downloader.download();
})
.version(packageJson.version);
pullCommand.addCommand(setCommand);
pullCommand.parse();
//# sourceMappingURL=cli.js.map

1
node_modules/ipull/dist/cli/cli.js.map generated vendored Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"cli.js","sourceRoot":"","sources":["../../src/cli/cli.ts"],"names":[],"mappings":";AACA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAC,OAAO,EAAE,MAAM,EAAC,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAC,WAAW,EAAC,MAAM,aAAa,CAAC;AACxC,OAAO,EAAC,YAAY,EAAE,gBAAgB,EAAC,MAAM,8BAA8B,CAAC;AAC5E,OAAO,EAAC,UAAU,EAAC,MAAM,mBAAmB,CAAC;AAC7C,OAAO,eAAe,EAAE,EAAC,mBAAmB,EAAE,YAAY,EAAC,MAAM,8BAA8B,CAAC;AAIhG,MAAM,WAAW,GAAG,IAAI,OAAO,EAAE,CAAC;AAClC,WAAW;KACN,WAAW,CAAC,oDAAoD,CAAC;KACjE,QAAQ,CAAC,YAAY,EAAE,oBAAoB,CAAC;KAC5C,MAAM,CAAC,kBAAkB,EAAE,gCAAgC,CAAC;KAC5D,MAAM,CAAC,2BAA2B,EAAE,gCAAgC,EAAE,GAAG,CAAC;KAC1E,SAAS,CAAC,IAAI,MAAM,CAAC,oBAAoB,EAAE,mCAAmC,CAAC,CAAC,OAAO,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC;KAC7H,SAAS,CAAC,IAAI,MAAM,CAAC,qBAAqB,EAAE,uBAAuB,CAAC,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC;KACnG,MAAM,CAAC,oBAAoB,EAAE,mEAAmE,CAAC;KACjG,MAAM,CAAC,KAAK,EAAE,QAAkB,EAAE,EAAE,EAAC,IAAI,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,EAAE,OAAO,EAAE,KAAK,EAM7F,EAAE,EAAE;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACrB,WAAW,CAAC,UAAU,EAAE,CAAC;QACzB,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IAED,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QACrB,MAAM,WAAW,GAAG,YAAY,IAAI,MAAM,mBAAmB,CAAC,YAAY,CAAC,CAAC;QAC5E,MAAM,SAAS,GAAG,WAAW,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,MAAM,eAAe,CAAC,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC;QACzF,MAAM,QAAQ,GAAG,WAAW,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC;QAEjF,OAAO,MAAM,YAAY,CAAC;YACtB,GAAG,EAAE,IAAI;YACT,SAAS;YACT,QAAQ;YACR,YAAY;YACZ,eAAe,EAAE,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC;YACpC,WAAW,EAAE,OAAc;SAC9B,CAAC,CAAC;IACP,CAAC,CAAC,CACL,CAAC;IAEF,MAAM,UAAU,GAAG,MAAM,gBAAgB,CAAC;QACtC,YAAY;QACZ,WAAW,EAAE,IAAI;QACjB,QAAQ,EAAE,KAAK;KAClB,EAAE,GAAG,aAAa,CAAC,CAAC;IACrB,MAAM,UAAU,CAAC,QAAQ,EAAE,CAAC;AAChC,CAAC,CAAC;KACD,OAAO,CAAC,WAAW,CAAC,OAAO,CAAC,CAAC;AAElC,WAAW,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC;AACnC,WAAW,CAAC,KAAK,EAAE,CAAC"}

2
node_modules/ipull/dist/cli/commands/set.d.ts generated vendored Normal file
View File

@@ -0,0 +1,2 @@
import { Command } from "commander";
export declare const setCommand: Command;

30
node_modules/ipull/dist/cli/commands/set.js generated vendored Normal file
View File

@@ -0,0 +1,30 @@
import { Command } from "commander";
import { AppDB } from "../../settings/settings.js";
const HELP_TEXT = `
You can set that file extensions will be download to specific path.
For example all zip files will be download to ~/Downloads/zip/:
pull set .zip ~/Downloads/zip/
You can set default download path:
pull set default ~/Downloads/
`;
export const setCommand = new Command("set");
setCommand.description("Set download locations")
.argument("[path]", "Path to the settings")
.argument("<value>", "Value to set")
.option("-d delete", "Delete the setting")
.addHelpText("afterAll", HELP_TEXT)
.action(async (path, value, { delete: deleteSetting }) => {
if (deleteSetting) {
await AppDB.update(data => {
delete data[path];
});
console.log(`Deleted ${path}`);
return;
}
await AppDB.update(data => {
data[path] = value;
});
console.log(`${value} set to ${path}`);
});
//# sourceMappingURL=set.js.map

1
node_modules/ipull/dist/cli/commands/set.js.map generated vendored Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"set.js","sourceRoot":"","sources":["../../../src/cli/commands/set.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,OAAO,EAAC,MAAM,WAAW,CAAC;AAClC,OAAO,EAAC,KAAK,EAAC,MAAM,4BAA4B,CAAC;AAEjD,MAAM,SAAS,GACX;;;;;;;EAOF,CAAC;AAEH,MAAM,CAAC,MAAM,UAAU,GAAG,IAAI,OAAO,CAAC,KAAK,CAAC,CAAC;AAE7C,UAAU,CAAC,WAAW,CAAC,wBAAwB,CAAC;KAC3C,QAAQ,CAAC,QAAQ,EAAE,sBAAsB,CAAC;KAC1C,QAAQ,CAAC,SAAS,EAAE,cAAc,CAAC;KACnC,MAAM,CAAC,WAAW,EAAE,oBAAoB,CAAC;KACzC,WAAW,CAAC,UAAU,EAAE,SAAS,CAAC;KAClC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,EAAC,MAAM,EAAE,aAAa,EAAC,EAAE,EAAE;IACnD,IAAI,aAAa,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;YACtB,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC;QACtB,CAAC,CAAC,CAAC;QAEH,OAAO,CAAC,GAAG,CAAC,WAAW,IAAI,EAAE,CAAC,CAAC;QAC/B,OAAO;IACX,CAAC;IAED,MAAM,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE;QACtB,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;IACvB,CAAC,CAAC,CAAC;IACH,OAAO,CAAC,GAAG,CAAC,GAAG,KAAK,WAAW,IAAI,EAAE,CAAC,CAAC;AAC3C,CAAC,CAAC,CAAC"}

View File

@@ -0,0 +1,3 @@
export default function findDownloadDir(fileName?: string): Promise<string>;
export declare function findFileName(url: string): string;
export declare function downloadToDirectory(path: string): Promise<boolean>;

27
node_modules/ipull/dist/cli/utils/find-download-dir.js generated vendored Normal file
View File

@@ -0,0 +1,27 @@
import path from "path";
import fs from "fs-extra";
import { AppDB } from "../../settings/settings.js";
const DEFAULT_DOWNLOAD_DIR = process.cwd();
export default async function findDownloadDir(fileName) {
const downloadLocation = AppDB.data[path.extname(fileName || "")];
const defaultLocation = AppDB.data["default"];
return downloadLocation || defaultLocation || DEFAULT_DOWNLOAD_DIR;
}
export function findFileName(url) {
try {
return path.basename(new URL(url).pathname);
}
catch {
return path.basename(url);
}
}
export async function downloadToDirectory(path) {
try {
const stats = await fs.lstat(path);
return stats.isDirectory();
}
catch {
return false;
}
}
//# sourceMappingURL=find-download-dir.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"find-download-dir.js","sourceRoot":"","sources":["../../../src/cli/utils/find-download-dir.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAC1B,OAAO,EAAC,KAAK,EAAC,MAAM,4BAA4B,CAAC;AAEjD,MAAM,oBAAoB,GAAG,OAAO,CAAC,GAAG,EAAE,CAAC;AAE3C,MAAM,CAAC,OAAO,CAAC,KAAK,UAAU,eAAe,CAAC,QAAiB;IAC3D,MAAM,gBAAgB,GAAG,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,CAAC;IAClE,MAAM,eAAe,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;IAC9C,OAAO,gBAAgB,IAAI,eAAe,IAAI,oBAAoB,CAAC;AACvE,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,GAAW;IACpC,IAAI,CAAC;QACD,OAAO,IAAI,CAAC,QAAQ,CAAC,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAChD,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC;IAC9B,CAAC;AACL,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,IAAY;IAClD,IAAI,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,EAAE,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACnC,OAAO,KAAK,CAAC,WAAW,EAAE,CAAC;IAC/B,CAAC;IAAC,MAAM,CAAC;QACL,OAAO,KAAK,CAAC;IACjB,CAAC;AACL,CAAC"}

3
node_modules/ipull/dist/const.d.ts generated vendored Normal file
View File

@@ -0,0 +1,3 @@
export declare const __dirname: string;
export declare const packageJson: any;
export declare const DB_PATH: string;

7
node_modules/ipull/dist/const.js generated vendored Normal file
View File

@@ -0,0 +1,7 @@
import { fileURLToPath } from "url";
import path from "path";
import fs from "fs-extra";
export const __dirname = fileURLToPath(new URL(".", import.meta.url));
export const packageJson = await fs.readJSON(path.join(__dirname, "..", "package.json"));
export const DB_PATH = path.join(__dirname, "..", "db.json");
//# sourceMappingURL=const.js.map

1
node_modules/ipull/dist/const.js.map generated vendored Normal file
View File

@@ -0,0 +1 @@
{"version":3,"file":"const.js","sourceRoot":"","sources":["../src/const.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,aAAa,EAAC,MAAM,KAAK,CAAC;AAClC,OAAO,IAAI,MAAM,MAAM,CAAC;AACxB,OAAO,EAAE,MAAM,UAAU,CAAC;AAE1B,MAAM,CAAC,MAAM,SAAS,GAAG,aAAa,CAAC,IAAI,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACtE,MAAM,CAAC,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,cAAc,CAAC,CAAC,CAAC;AAEzF,MAAM,CAAC,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC"}

14
node_modules/ipull/dist/download/browser-download.d.ts generated vendored Normal file
View File

@@ -0,0 +1,14 @@
import DownloadEngineBrowser, { DownloadEngineOptionsBrowser } from "./download-engine/engine/download-engine-browser.js";
import DownloadEngineMultiDownload from "./download-engine/engine/download-engine-multi-download.js";
export type DownloadFileBrowserOptions = DownloadEngineOptionsBrowser & {
/** @deprecated use partURLs instead */
partsURL?: string[];
};
/**
* Download one file in the browser environment.
*/
export declare function downloadFileBrowser(options: DownloadFileBrowserOptions): Promise<DownloadEngineBrowser<import("./download-engine/streams/download-engine-write-stream/download-engine-write-stream-browser.js").default>>;
/**
* Download multiple files in the browser environment.
*/
export declare function downloadSequenceBrowser(...downloads: (DownloadEngineBrowser | Promise<DownloadEngineBrowser>)[]): Promise<DownloadEngineMultiDownload<import("../index.js").BaseDownloadEngine>>;

25
node_modules/ipull/dist/download/browser-download.js generated vendored Normal file
View File

@@ -0,0 +1,25 @@
import DownloadEngineBrowser from "./download-engine/engine/download-engine-browser.js";
import DownloadEngineMultiDownload from "./download-engine/engine/download-engine-multi-download.js";
import { NoDownloadEngineProvidedError } from "./download-engine/engine/error/no-download-engine-provided-error.js";
const DEFAULT_PARALLEL_STREAMS_FOR_BROWSER = 3;
/**
* Download one file in the browser environment.
*/
export async function downloadFileBrowser(options) {
// TODO: Remove in the next major version
if (!("url" in options) && options.partsURL) {
options.partURLs ??= options.partsURL;
}
options.parallelStreams ??= DEFAULT_PARALLEL_STREAMS_FOR_BROWSER;
return await DownloadEngineBrowser.createFromOptions(options);
}
/**
* Download multiple files in the browser environment.
*/
export async function downloadSequenceBrowser(...downloads) {
if (downloads.length === 0) {
throw new NoDownloadEngineProvidedError();
}
return await DownloadEngineMultiDownload.fromEngines(downloads);
}
//# sourceMappingURL=browser-download.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"browser-download.js","sourceRoot":"","sources":["../../src/download/browser-download.ts"],"names":[],"mappings":"AAAA,OAAO,qBAAqD,MAAM,qDAAqD,CAAC;AACxH,OAAO,2BAA2B,MAAM,4DAA4D,CAAC;AACrG,OAAO,EAAC,6BAA6B,EAAC,MAAM,qEAAqE,CAAC;AAElH,MAAM,oCAAoC,GAAG,CAAC,CAAC;AAO/C;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,mBAAmB,CAAC,OAAmC;IACzE,yCAAyC;IACzC,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QAC1C,OAAO,CAAC,QAAQ,KAAK,OAAO,CAAC,QAAQ,CAAC;IAC1C,CAAC;IAED,OAAO,CAAC,eAAe,KAAK,oCAAoC,CAAC;IACjE,OAAO,MAAM,qBAAqB,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;AAClE,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,GAAG,SAAqE;IAClH,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,6BAA6B,EAAE,CAAC;IAC9C,CAAC;IAED,OAAO,MAAM,2BAA2B,CAAC,WAAW,CAAC,SAAS,CAAC,CAAC;AACpE,CAAC"}

View File

@@ -0,0 +1,69 @@
import ProgressStatusFile, { DownloadStatus, ProgressStatus } from "./progress-status-file.js";
import { DownloadFile, SaveProgressInfo } from "../types.js";
import BaseDownloadEngineFetchStream from "../streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js";
import BaseDownloadEngineWriteStream from "../streams/download-engine-write-stream/base-download-engine-write-stream.js";
import retry from "async-retry";
import { EventEmitter } from "eventemitter3";
import { AvailablePrograms } from "./download-programs/switch-program.js";
import BaseDownloadProgram from "./download-programs/base-download-program.js";
export type DownloadEngineFileOptions = {
chunkSize?: number;
parallelStreams?: number;
retry?: retry.Options;
comment?: string;
fetchStream: BaseDownloadEngineFetchStream;
writeStream: BaseDownloadEngineWriteStream;
onFinishAsync?: () => Promise<void>;
onStartedAsync?: () => Promise<void>;
onCloseAsync?: () => Promise<void>;
onSaveProgressAsync?: (progress: SaveProgressInfo) => Promise<void>;
programType?: AvailablePrograms;
};
export type DownloadEngineFileOptionsWithDefaults = DownloadEngineFileOptions & {
chunkSize: number;
parallelStreams: number;
};
export type DownloadEngineFileEvents = {
start: () => void;
paused: () => void;
resumed: () => void;
progress: (progress: ProgressStatus) => void;
save: (progress: SaveProgressInfo) => void;
finished: () => void;
closed: () => void;
[key: string]: any;
};
export default class DownloadEngineFile extends EventEmitter<DownloadEngineFileEvents> {
readonly file: DownloadFile;
options: DownloadEngineFileOptionsWithDefaults;
protected _progress: SaveProgressInfo;
protected _closed: boolean;
protected _progressStatus: ProgressStatusFile;
protected _activeStreamBytes: {
[key: number]: number;
};
protected _activeProgram?: BaseDownloadProgram;
protected _downloadStatus: DownloadStatus;
private _latestProgressDate;
constructor(file: DownloadFile, options: DownloadEngineFileOptions);
private _createProgressFlags;
get downloadSize(): number;
get fileName(): string;
get status(): ProgressStatus;
protected get _activePart(): import("../types.js").DownloadFilePart;
private get _downloadedPartsSize();
private get _activeDownloadedChunkSize();
get transferredBytes(): number;
protected _emptyChunksForPart(part: number): any[];
private _initEventReloadStatus;
private _initProgress;
download(): Promise<void>;
protected _downloadSlice(startChunk: number, endChunk: number): Promise<void>;
protected _saveProgress(): Promise<void> | undefined;
protected _sendProgressDownloadPart(): void;
pause(): void;
resume(): void;
close(): Promise<void>;
finished(comment?: string): void;
[Symbol.dispose](): Promise<void>;
}

View File

@@ -0,0 +1,261 @@
import ProgressStatusFile, { DownloadFlags, DownloadStatus } from "./progress-status-file.js";
import { ChunkStatus } from "../types.js";
import { EventEmitter } from "eventemitter3";
import { withLock } from "lifecycle-utils";
import switchProgram from "./download-programs/switch-program.js";
import { pushComment } from "./utils/push-comment.js";
const DEFAULT_OPTIONS = {
chunkSize: 1024 * 1024 * 5,
parallelStreams: 1
};
export default class DownloadEngineFile extends EventEmitter {
file;
options;
_progress = {
part: 0,
chunks: [],
chunkSize: 0,
parallelStreams: 0
};
_closed = false;
_progressStatus;
_activeStreamBytes = {};
_activeProgram;
_downloadStatus = DownloadStatus.NotStarted;
_latestProgressDate = 0;
constructor(file, options) {
super();
this.file = file;
this.options = { ...DEFAULT_OPTIONS, ...options };
this._progressStatus = new ProgressStatusFile(file.parts.length, file.localFileName, options.fetchStream.transferAction, this._createProgressFlags());
this._initProgress();
}
_createProgressFlags() {
const flags = [];
if (this.options.skipExisting) {
flags.push(DownloadFlags.Existing);
}
return flags;
}
get downloadSize() {
return this.file.parts.reduce((acc, part) => acc + part.size, 0);
}
get fileName() {
return this.file.localFileName;
}
get status() {
return this._progressStatus.createStatus(this._progress.part + 1, this.transferredBytes, this.downloadSize, this._downloadStatus, this.options.comment);
}
get _activePart() {
return this.file.parts[this._progress.part];
}
get _downloadedPartsSize() {
return this.file.parts.slice(0, this._progress.part)
.reduce((acc, part) => acc + part.size, 0);
}
get _activeDownloadedChunkSize() {
return this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE).length * this._progress.chunkSize;
}
get transferredBytes() {
if (this._downloadStatus === DownloadStatus.Finished) {
return this.downloadSize;
}
const streamingBytes = Object.values(this._activeStreamBytes)
.reduce((acc, bytes) => acc + bytes, 0);
const streamBytes = this._activeDownloadedChunkSize + streamingBytes;
const streamBytesMin = Math.min(streamBytes, this._activePart.size || streamBytes);
const allBytes = streamBytesMin + this._downloadedPartsSize;
return Math.min(allBytes, this.downloadSize || allBytes);
}
_emptyChunksForPart(part) {
const partInfo = this.file.parts[part];
if (partInfo.size === 0) {
return [ChunkStatus.NOT_STARTED];
}
const chunksCount = Math.ceil(partInfo.size / this.options.chunkSize);
return new Array(chunksCount).fill(ChunkStatus.NOT_STARTED);
}
_initEventReloadStatus() {
if (this._progress.part === this.file.parts.length - 1 && this._progress.chunks.every(c => c === ChunkStatus.COMPLETE)) {
this._downloadStatus = DownloadStatus.Finished;
}
}
_initProgress() {
if (this.options.skipExisting) {
this._progress.part = this.file.parts.length - 1;
this._downloadStatus = DownloadStatus.Finished;
this.options.comment = pushComment("Skipping existing", this.options.comment);
}
else if (this.file.downloadProgress) {
this._progress = this.file.downloadProgress;
this._initEventReloadStatus();
}
else {
this._progress = {
part: 0,
chunks: this._emptyChunksForPart(0),
chunkSize: this.options.chunkSize,
parallelStreams: this.options.parallelStreams
};
}
}
async download() {
if (this._downloadStatus === DownloadStatus.NotStarted) {
this._downloadStatus = DownloadStatus.Active;
}
this._progressStatus.started();
this.emit("start");
await this.options.onStartedAsync?.();
for (let i = this._progress.part; i < this.file.parts.length && this._downloadStatus !== DownloadStatus.Finished; i++) {
if (this._closed)
return;
// If we are starting a new part, we need to reset the progress
if (i > this._progress.part || !this._activePart.acceptRange) {
this._progress.part = i;
this._progress.chunkSize = this.options.chunkSize;
this._progress.parallelStreams = this.options.parallelStreams;
this._progress.chunks = this._emptyChunksForPart(i);
}
// Reset in progress chunks
this._progress.chunks = this._progress.chunks.map(chunk => (chunk === ChunkStatus.COMPLETE ? ChunkStatus.COMPLETE : ChunkStatus.NOT_STARTED));
// Reset active stream progress
this._activeStreamBytes = {};
if (this._activePart.acceptRange) {
this._activeProgram = switchProgram(this._progress, this._downloadSlice.bind(this), this.options.fetchStream.programType || this.options.programType);
await this._activeProgram.download();
}
else {
const chunksToRead = this._activePart.size > 0 ? this._progress.chunks.length : Infinity;
await this._downloadSlice(0, chunksToRead);
}
}
// All parts are downloaded, we can clear the progress
this._activeStreamBytes = {};
this._latestProgressDate = 0;
if (this._closed)
return;
this._progressStatus.finished();
this._downloadStatus = DownloadStatus.Finished;
this._sendProgressDownloadPart();
this.emit("finished");
await this.options.onFinishAsync?.();
}
_downloadSlice(startChunk, endChunk) {
const fetchState = this.options.fetchStream.withSubState({
chunkSize: this._progress.chunkSize,
startChunk,
endChunk,
totalSize: this._activePart.size,
url: this._activePart.downloadURL,
rangeSupport: this._activePart.acceptRange,
onProgress: (length) => {
this._activeStreamBytes[startChunk] = length;
this._sendProgressDownloadPart();
}
});
const downloadedPartsSize = this._downloadedPartsSize;
this._progress.chunks[startChunk] = ChunkStatus.IN_PROGRESS;
return (async () => {
const allWrites = new Set();
let lastChunkSize = 0;
await fetchState.fetchChunks((chunks, writePosition, index) => {
if (this._closed || this._progress.chunks[index] != ChunkStatus.IN_PROGRESS) {
return;
}
for (const chunk of chunks) {
const writePromise = this.options.writeStream.write(downloadedPartsSize + writePosition, chunk);
writePosition += chunk.length;
if (writePromise) {
allWrites.add(writePromise);
writePromise.then(() => {
allWrites.delete(writePromise);
});
}
}
// if content length is 0, we do not know how many chunks we should have
if (this._activePart.size === 0) {
this._progress.chunks.push(ChunkStatus.NOT_STARTED);
}
this._progress.chunks[index] = ChunkStatus.COMPLETE;
lastChunkSize = chunks.reduce((last, current) => last + current.length, 0);
delete this._activeStreamBytes[startChunk];
void this._saveProgress();
const nextChunk = this._progress.chunks[index + 1];
const shouldReadNext = endChunk - index > 1; // grater than 1, meaning there is a next chunk
if (shouldReadNext) {
if (nextChunk == null || nextChunk != ChunkStatus.NOT_STARTED) {
return fetchState.close();
}
this._progress.chunks[index + 1] = ChunkStatus.IN_PROGRESS;
}
});
// On dynamic content length, we need to adjust the last chunk size
if (this._activePart.size === 0) {
this._activePart.size = this._activeDownloadedChunkSize - this.options.chunkSize + lastChunkSize;
this._progress.chunks = this._progress.chunks.filter(c => c === ChunkStatus.COMPLETE);
}
delete this._activeStreamBytes[startChunk];
await Promise.all(allWrites);
})();
}
_saveProgress() {
const thisProgress = this._latestProgressDate = Date.now();
this._sendProgressDownloadPart();
if (!this._activePart.acceptRange)
return;
this.emit("save", this._progress);
return withLock(this, "_saveLock", async () => {
if (thisProgress === this._latestProgressDate) {
await this.options.onSaveProgressAsync?.(this._progress);
}
});
}
_sendProgressDownloadPart() {
if (this._closed)
return;
this.emit("progress", this.status);
}
pause() {
if (this.options.fetchStream.paused) {
return;
}
this._downloadStatus = DownloadStatus.Paused;
this.options.fetchStream.emit("paused");
this.emit("paused");
this._sendProgressDownloadPart();
}
resume() {
if (!this.options.fetchStream.paused) {
return;
}
this._downloadStatus = DownloadStatus.Active;
this.options.fetchStream.emit("resumed");
this.emit("resumed");
this._sendProgressDownloadPart();
}
async close() {
if (this._closed)
return;
if (this._downloadStatus !== DownloadStatus.Finished) {
this._progressStatus.finished();
this._downloadStatus = DownloadStatus.Cancelled;
this._sendProgressDownloadPart();
}
this._closed = true;
this._activeProgram?.abort();
await this.options.onCloseAsync?.();
await this.options.writeStream.close();
await this.options.fetchStream.close();
this.emit("closed");
}
finished(comment) {
if (comment) {
this.options.comment = pushComment(comment, this.options.comment);
}
this._downloadStatus = DownloadStatus.Finished;
}
[Symbol.dispose]() {
return this.close();
}
}
//# sourceMappingURL=download-engine-file.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,15 @@
import { SaveProgressInfo } from "../../types.js";
export type ProgramSlice = {
start: number;
end: number;
};
export type DownloadSlice = (startChunk: number, endChunk: number) => Promise<void>;
export default abstract class BaseDownloadProgram {
protected savedProgress: SaveProgressInfo;
protected readonly _downloadSlice: DownloadSlice;
protected _aborted: boolean;
protected constructor(_savedProgress: SaveProgressInfo, _downloadSlice: DownloadSlice);
download(): Promise<void>;
protected abstract _createOneSlice(): ProgramSlice | null;
abort(): void;
}

View File

@@ -0,0 +1,38 @@
export default class BaseDownloadProgram {
savedProgress;
_downloadSlice;
_aborted = false;
constructor(_savedProgress, _downloadSlice) {
this._downloadSlice = _downloadSlice;
this.savedProgress = _savedProgress;
}
async download() {
if (this.savedProgress.parallelStreams === 1) {
return await this._downloadSlice(0, this.savedProgress.chunks.length);
}
const activeDownloads = [];
// eslint-disable-next-line no-constant-condition
while (true) {
while (activeDownloads.length >= this.savedProgress.parallelStreams) {
if (this._aborted)
return;
await Promise.race(activeDownloads);
}
const slice = this._createOneSlice();
if (slice == null)
break;
if (this._aborted)
return;
const promise = this._downloadSlice(slice.start, slice.end);
activeDownloads.push(promise);
promise.then(() => {
activeDownloads.splice(activeDownloads.indexOf(promise), 1);
});
}
await Promise.all(activeDownloads);
}
abort() {
this._aborted = true;
}
}
//# sourceMappingURL=base-download-program.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"base-download-program.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/download-file/download-programs/base-download-program.ts"],"names":[],"mappings":"AASA,MAAM,CAAC,OAAO,OAAgB,mBAAmB;IACnC,aAAa,CAAmB;IACvB,cAAc,CAAgB;IACvC,QAAQ,GAAG,KAAK,CAAC;IAE3B,YAAsB,cAAgC,EAAE,cAA6B;QACjF,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,aAAa,GAAG,cAAc,CAAC;IACxC,CAAC;IAEM,KAAK,CAAC,QAAQ;QACjB,IAAI,IAAI,CAAC,aAAa,CAAC,eAAe,KAAK,CAAC,EAAE,CAAC;YAC3C,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,CAAC,EAAE,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC1E,CAAC;QAED,MAAM,eAAe,GAAmB,EAAE,CAAC;QAE3C,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACV,OAAO,eAAe,CAAC,MAAM,IAAI,IAAI,CAAC,aAAa,CAAC,eAAe,EAAE,CAAC;gBAClE,IAAI,IAAI,CAAC,QAAQ;oBAAE,OAAO;gBAC1B,MAAM,OAAO,CAAC,IAAI,CAAC,eAAe,CAAC,CAAC;YACxC,CAAC;YAED,MAAM,KAAK,GAAG,IAAI,CAAC,eAAe,EAAE,CAAC;YACrC,IAAI,KAAK,IAAI,IAAI;gBAAE,MAAM;YAEzB,IAAI,IAAI,CAAC,QAAQ;gBAAE,OAAO;YAC1B,MAAM,OAAO,GAAG,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,KAAK,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC;YAC5D,eAAe,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC9B,OAAO,CAAC,IAAI,CAAC,GAAG,EAAE;gBACd,eAAe,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC;YAChE,CAAC,CAAC,CAAC;QACP,CAAC;QAED,MAAM,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;IACvC,CAAC;IAIM,KAAK;QACR,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC;IACzB,CAAC;CACJ"}

View File

@@ -0,0 +1,6 @@
import { SaveProgressInfo } from "../../types.js";
import BaseDownloadProgram, { DownloadSlice, ProgramSlice } from "./base-download-program.js";
export default class DownloadProgramChunks extends BaseDownloadProgram {
constructor(savedProgress: SaveProgressInfo, downloadSlice: DownloadSlice);
protected _createOneSlice(): ProgramSlice | null;
}

View File

@@ -0,0 +1,14 @@
import { ChunkStatus } from "../../types.js";
import BaseDownloadProgram from "./base-download-program.js";
export default class DownloadProgramChunks extends BaseDownloadProgram {
constructor(savedProgress, downloadSlice) {
super(savedProgress, downloadSlice);
}
_createOneSlice() {
const notDownloadedIndex = this.savedProgress.chunks.findIndex(c => c === ChunkStatus.NOT_STARTED);
if (notDownloadedIndex === -1)
return null;
return { start: notDownloadedIndex, end: notDownloadedIndex + 1 };
}
}
//# sourceMappingURL=download-program-chunks.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"download-program-chunks.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/download-file/download-programs/download-program-chunks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAmB,MAAM,gBAAgB,CAAC;AAC7D,OAAO,mBAAkD,MAAM,4BAA4B,CAAC;AAE5F,MAAM,CAAC,OAAO,OAAO,qBAAsB,SAAQ,mBAAmB;IAClE,YAAmB,aAA+B,EAAE,aAA4B;QAC5E,KAAK,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;IACxC,CAAC;IAES,eAAe;QACrB,MAAM,kBAAkB,GAAG,IAAI,CAAC,aAAa,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,WAAW,CAAC,WAAW,CAAC,CAAC;QACnG,IAAI,kBAAkB,KAAK,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QAE3C,OAAO,EAAC,KAAK,EAAE,kBAAkB,EAAE,GAAG,EAAE,kBAAkB,GAAG,CAAC,EAAC,CAAC;IACpE,CAAC;CACJ"}

View File

@@ -0,0 +1,7 @@
import { SaveProgressInfo } from "../../types.js";
import BaseDownloadProgram, { DownloadSlice, ProgramSlice } from "./base-download-program.js";
export default class DownloadProgramStream extends BaseDownloadProgram {
constructor(savedProgress: SaveProgressInfo, downloadSlice: DownloadSlice);
protected _createOneSlice(): ProgramSlice | null;
private _findChunksSlices;
}

View File

@@ -0,0 +1,36 @@
import { ChunkStatus } from "../../types.js";
import BaseDownloadProgram from "./base-download-program.js";
export default class DownloadProgramStream extends BaseDownloadProgram {
constructor(savedProgress, downloadSlice) {
super(savedProgress, downloadSlice);
}
_createOneSlice() {
const slice = this._findChunksSlices()[0];
if (!slice)
return null;
const length = slice.end - slice.start;
return { start: Math.floor(slice.start + length / 2), end: slice.end };
}
_findChunksSlices() {
const chunksSlices = [];
let start = 0;
let currentIndex = 0;
for (const chunk of this.savedProgress.chunks) {
if (chunk !== ChunkStatus.NOT_STARTED) {
if (start === currentIndex) {
start = ++currentIndex;
continue;
}
chunksSlices.push({ start, end: currentIndex });
start = ++currentIndex;
continue;
}
currentIndex++;
}
if (start !== currentIndex) {
chunksSlices.push({ start, end: currentIndex });
}
return chunksSlices.sort((a, b) => (b.end - b.start) - (a.end - a.start));
}
}
//# sourceMappingURL=download-program-stream.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"download-program-stream.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/download-file/download-programs/download-program-stream.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,WAAW,EAAmB,MAAM,gBAAgB,CAAC;AAC7D,OAAO,mBAAkD,MAAM,4BAA4B,CAAC;AAG5F,MAAM,CAAC,OAAO,OAAO,qBAAsB,SAAQ,mBAAmB;IAClE,YAAmB,aAA+B,EAAE,aAA4B;QAC5E,KAAK,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;IACxC,CAAC;IAES,eAAe;QACrB,MAAM,KAAK,GAAG,IAAI,CAAC,iBAAiB,EAAE,CAAC,CAAC,CAAC,CAAC;QAC1C,IAAI,CAAC,KAAK;YAAE,OAAO,IAAI,CAAC;QACxB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,GAAG,KAAK,CAAC,KAAK,CAAC;QACvC,OAAO,EAAC,KAAK,EAAE,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,MAAM,GAAG,CAAC,CAAC,EAAE,GAAG,EAAE,KAAK,CAAC,GAAG,EAAC,CAAC;IACzE,CAAC;IAEO,iBAAiB;QACrB,MAAM,YAAY,GAAmB,EAAE,CAAC;QAExC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,YAAY,GAAG,CAAC,CAAC;QACrB,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC5C,IAAI,KAAK,KAAK,WAAW,CAAC,WAAW,EAAE,CAAC;gBACpC,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;oBACzB,KAAK,GAAG,EAAE,YAAY,CAAC;oBACvB,SAAS;gBACb,CAAC;gBACD,YAAY,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,GAAG,EAAE,YAAY,EAAC,CAAC,CAAC;gBAC9C,KAAK,GAAG,EAAE,YAAY,CAAC;gBACvB,SAAS;YACb,CAAC;YAED,YAAY,EAAE,CAAC;QACnB,CAAC;QAED,IAAI,KAAK,KAAK,YAAY,EAAE,CAAC;YACzB,YAAY,CAAC,IAAI,CAAC,EAAC,KAAK,EAAE,GAAG,EAAE,YAAY,EAAC,CAAC,CAAC;QAClD,CAAC;QAED,OAAO,YAAY,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC;IAC9E,CAAC;CACJ"}

View File

@@ -0,0 +1,6 @@
import { SaveProgressInfo } from "../../types.js";
import { DownloadSlice } from "./base-download-program.js";
import DownloadProgramChunks from "./download-program-chunks.js";
import DownloadProgramStream from "./download-program-stream.js";
export type AvailablePrograms = "stream" | "chunks";
export default function switchProgram(savedProgress: SaveProgressInfo, downloadSlice: DownloadSlice, name?: AvailablePrograms): DownloadProgramChunks | DownloadProgramStream;

View File

@@ -0,0 +1,12 @@
import DownloadProgramChunks from "./download-program-chunks.js";
import DownloadProgramStream from "./download-program-stream.js";
export default function switchProgram(savedProgress, downloadSlice, name) {
switch (name) {
case "chunks":
return new DownloadProgramChunks(savedProgress, downloadSlice);
case "stream":
default:
return new DownloadProgramStream(savedProgress, downloadSlice);
}
}
//# sourceMappingURL=switch-program.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"switch-program.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/download-file/download-programs/switch-program.ts"],"names":[],"mappings":"AAEA,OAAO,qBAAqB,MAAM,8BAA8B,CAAC;AACjE,OAAO,qBAAqB,MAAM,8BAA8B,CAAC;AAIjE,MAAM,CAAC,OAAO,UAAU,aAAa,CAAC,aAA+B,EAAE,aAA4B,EAAE,IAAwB;IACzH,QAAQ,IAAI,EAAE,CAAC;QACX,KAAK,QAAQ;YACT,OAAO,IAAI,qBAAqB,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;QACnE,KAAK,QAAQ,CAAC;QACd;YACI,OAAO,IAAI,qBAAqB,CAAC,aAAa,EAAE,aAAa,CAAC,CAAC;IACvE,CAAC;AACL,CAAC"}

View File

@@ -0,0 +1,43 @@
export type ProgressStatus = {
totalBytes: number;
totalDownloadParts: number;
fileName: string;
comment?: string;
downloadPart: number;
transferredBytes: number;
startTime: number;
endTime: number;
transferAction: string;
downloadStatus: DownloadStatus;
downloadFlags: DownloadFlags[];
};
export declare enum DownloadStatus {
Loading = "Loading",
Active = "Active",
Paused = "Paused",
NotStarted = "NotStarted",
Finished = "Finished",
Cancelled = "Cancelled",
Error = "Error"
}
export declare enum DownloadFlags {
Existing = "Existing",
DownloadSequence = "DownloadSequence"
}
export default class ProgressStatusFile {
readonly totalDownloadParts: number;
readonly fileName: string;
readonly comment?: string;
readonly downloadPart: number;
readonly transferredBytes: number;
readonly transferAction: string;
downloadStatus: DownloadStatus;
downloadFlags: DownloadFlags[];
totalBytes: number;
startTime: number;
endTime: number;
constructor(totalDownloadParts: number, fileName: string, transferAction?: string, downloadFlags?: DownloadFlags[], comment?: string, downloadPart?: number, transferredBytes?: number, downloadStatus?: DownloadStatus);
started(): void;
finished(): void;
createStatus(downloadPart: number, transferredBytes: number, totalBytes?: number, downloadStatus?: DownloadStatus, comment?: string | undefined): ProgressStatusFile;
}

View File

@@ -0,0 +1,52 @@
export var DownloadStatus;
(function (DownloadStatus) {
DownloadStatus["Loading"] = "Loading";
DownloadStatus["Active"] = "Active";
DownloadStatus["Paused"] = "Paused";
DownloadStatus["NotStarted"] = "NotStarted";
DownloadStatus["Finished"] = "Finished";
DownloadStatus["Cancelled"] = "Cancelled";
DownloadStatus["Error"] = "Error";
})(DownloadStatus || (DownloadStatus = {}));
export var DownloadFlags;
(function (DownloadFlags) {
DownloadFlags["Existing"] = "Existing";
DownloadFlags["DownloadSequence"] = "DownloadSequence";
})(DownloadFlags || (DownloadFlags = {}));
export default class ProgressStatusFile {
totalDownloadParts;
fileName;
comment;
downloadPart;
transferredBytes;
transferAction;
downloadStatus = DownloadStatus.Active;
downloadFlags = [];
totalBytes = 0;
startTime = 0;
endTime = 0;
constructor(totalDownloadParts, fileName, transferAction = "Transferring", downloadFlags = [], comment, downloadPart = 0, transferredBytes = 0, downloadStatus = DownloadStatus.Active) {
this.transferAction = transferAction;
this.transferredBytes = transferredBytes;
this.downloadPart = downloadPart;
this.comment = comment;
this.fileName = fileName;
this.totalDownloadParts = totalDownloadParts;
this.downloadFlags = downloadFlags;
this.downloadStatus = downloadStatus;
}
started() {
this.startTime = Date.now();
}
finished() {
this.endTime = Date.now();
}
createStatus(downloadPart, transferredBytes, totalBytes = this.totalBytes, downloadStatus = DownloadStatus.Active, comment = this.comment) {
const newStatus = new ProgressStatusFile(this.totalDownloadParts, this.fileName, this.transferAction, this.downloadFlags, comment, downloadPart, transferredBytes, downloadStatus);
newStatus.totalBytes = totalBytes;
newStatus.startTime = this.startTime;
newStatus.endTime = this.endTime;
return newStatus;
}
}
//# sourceMappingURL=progress-status-file.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"progress-status-file.js","sourceRoot":"","sources":["../../../../src/download/download-engine/download-file/progress-status-file.ts"],"names":[],"mappings":"AAcA,MAAM,CAAN,IAAY,cAQX;AARD,WAAY,cAAc;IACtB,qCAAmB,CAAA;IACnB,mCAAiB,CAAA;IACjB,mCAAiB,CAAA;IACjB,2CAAyB,CAAA;IACzB,uCAAqB,CAAA;IACrB,yCAAuB,CAAA;IACvB,iCAAe,CAAA;AACnB,CAAC,EARW,cAAc,KAAd,cAAc,QAQzB;AAED,MAAM,CAAN,IAAY,aAGX;AAHD,WAAY,aAAa;IACrB,sCAAqB,CAAA;IACrB,sDAAqC,CAAA;AACzC,CAAC,EAHW,aAAa,KAAb,aAAa,QAGxB;AAED,MAAM,CAAC,OAAO,OAAO,kBAAkB;IACnB,kBAAkB,CAAS;IAC3B,QAAQ,CAAS;IACjB,OAAO,CAAU;IACjB,YAAY,CAAS;IACrB,gBAAgB,CAAS;IACzB,cAAc,CAAS;IAChC,cAAc,GAAmB,cAAc,CAAC,MAAM,CAAC;IACvD,aAAa,GAAoB,EAAE,CAAC;IACpC,UAAU,GAAW,CAAC,CAAC;IACvB,SAAS,GAAW,CAAC,CAAC;IACtB,OAAO,GAAW,CAAC,CAAC;IAE3B,YACI,kBAA0B,EAC1B,QAAgB,EAChB,cAAc,GAAG,cAAc,EAC/B,gBAAiC,EAAE,EACnC,OAAgB,EAChB,YAAY,GAAG,CAAC,EAChB,gBAAgB,GAAG,CAAC,EACpB,cAAc,GAAG,cAAc,CAAC,MAAM;QAEtC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,gBAAgB,GAAG,gBAAgB,CAAC;QACzC,IAAI,CAAC,YAAY,GAAG,YAAY,CAAC;QACjC,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,kBAAkB,GAAG,kBAAkB,CAAC;QAC7C,IAAI,CAAC,aAAa,GAAG,aAAa,CAAC;QACnC,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;IACzC,CAAC;IAEM,OAAO;QACV,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAChC,CAAC;IAEM,QAAQ;QACX,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC9B,CAAC;IAEM,YAAY,CAAC,YAAoB,EAAE,gBAAwB,EAAE,UAAU,GAAG,IAAI,CAAC,UAAU,EAAE,cAAc,GAAG,cAAc,CAAC,MAAM,EAAE,OAAO,GAAG,IAAI,CAAC,OAAO;QAC5J,MAAM,SAAS,GAAG,IAAI,kBAAkB,CACpC,IAAI,CAAC,kBAAkB,EACvB,IAAI,CAAC,QAAQ,EACb,IAAI,CAAC,cAAc,EACnB,IAAI,CAAC,aAAa,EAClB,OAAO,EACP,YAAY,EACZ,gBAAgB,EAChB,cAAc,CACjB,CAAC;QAEF,SAAS,CAAC,UAAU,GAAG,UAAU,CAAC;QAClC,SAAS,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QACrC,SAAS,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;QAEjC,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ"}

View File

@@ -0,0 +1 @@
export declare function pushComment(newComment: string, comment?: string): string;

View File

@@ -0,0 +1,7 @@
export function pushComment(newComment, comment = "") {
if (comment.length) {
return `${newComment}, ${comment}`;
}
return newComment;
}
//# sourceMappingURL=push-comment.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"push-comment.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/download-file/utils/push-comment.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,WAAW,CAAC,UAAkB,EAAE,OAAO,GAAG,EAAE;IACxD,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;QACjB,OAAO,GAAG,UAAU,KAAK,OAAO,EAAE,CAAC;IACvC,CAAC;IACD,OAAO,UAAU,CAAC;AACtB,CAAC"}

View File

@@ -0,0 +1,50 @@
import { DownloadFile, SaveProgressInfo } from "../types.js";
import DownloadEngineFile, { DownloadEngineFileOptions } from "../download-file/download-engine-file.js";
import BaseDownloadEngineFetchStream, { BaseDownloadEngineFetchStreamOptions } from "../streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js";
import { EventEmitter } from "eventemitter3";
import ProgressStatisticsBuilder, { ProgressStatusWithIndex } from "../../transfer-visualize/progress-statistics-builder.js";
import retry from "async-retry";
import { AvailablePrograms } from "../download-file/download-programs/switch-program.js";
export type InputURLOptions = {
partURLs: string[];
} | {
url: string;
};
export type BaseDownloadEngineOptions = InputURLOptions & BaseDownloadEngineFetchStreamOptions & {
chunkSize?: number;
parallelStreams?: number;
retry?: retry.Options;
comment?: string;
programType?: AvailablePrograms;
};
export type BaseDownloadEngineEvents = {
start: () => void;
paused: () => void;
resumed: () => void;
progress: (progress: ProgressStatusWithIndex) => void;
save: (progress: SaveProgressInfo) => void;
finished: () => void;
closed: () => void;
[key: string]: any;
};
export default class BaseDownloadEngine extends EventEmitter<BaseDownloadEngineEvents> {
readonly options: DownloadEngineFileOptions;
protected readonly _engine: DownloadEngineFile;
protected _progressStatisticsBuilder: ProgressStatisticsBuilder;
protected _downloadStarted: boolean;
protected _latestStatus?: ProgressStatusWithIndex;
protected constructor(engine: DownloadEngineFile, options: DownloadEngineFileOptions);
get file(): DownloadFile;
get downloadSize(): number;
get fileName(): string;
get status(): import("../../transfer-visualize/format-transfer-status.js").FormattedStatus;
get downloadStatues(): import("../../transfer-visualize/format-transfer-status.js").FormattedStatus[];
protected _initEvents(): void;
download(): Promise<void>;
pause(): void;
resume(): void;
close(): Promise<void>;
protected static _createDownloadFile(parts: string[], fetchStream: BaseDownloadEngineFetchStream): Promise<DownloadFile>;
protected static _validateURL(options: InputURLOptions): void;
protected static _validateOptions(options: BaseDownloadEngineOptions): void;
}

View File

@@ -0,0 +1,140 @@
import UrlInputError from "./error/url-input-error.js";
import { EventEmitter } from "eventemitter3";
import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js";
import DownloadAlreadyStartedError from "./error/download-already-started-error.js";
import StatusCodeError from "../streams/download-engine-fetch-stream/errors/status-code-error.js";
import { InvalidOptionError } from "./error/InvalidOptionError.js";
const IGNORE_HEAD_STATUS_CODES = [405, 501, 404];
export default class BaseDownloadEngine extends EventEmitter {
options;
_engine;
_progressStatisticsBuilder = new ProgressStatisticsBuilder();
_downloadStarted = false;
_latestStatus;
constructor(engine, options) {
super();
this.options = options;
this._engine = engine;
this._progressStatisticsBuilder.add(engine);
this._initEvents();
}
get file() {
return this._engine.file;
}
get downloadSize() {
return this._engine.downloadSize;
}
get fileName() {
return this.file.localFileName;
}
get status() {
return this._latestStatus ?? ProgressStatisticsBuilder.oneStatistics(this._engine);
}
get downloadStatues() {
return [this.status];
}
/**
* @internal
*/
get _fileEngineOptions() {
return this._engine.options;
}
_initEvents() {
this._engine.on("start", () => {
return this.emit("start");
});
this._engine.on("save", (info) => {
return this.emit("save", info);
});
this._engine.on("finished", () => {
return this.emit("finished");
});
this._engine.on("closed", () => {
return this.emit("closed");
});
this._engine.on("paused", () => {
return this.emit("paused");
});
this._engine.on("resumed", () => {
return this.emit("resumed");
});
this._progressStatisticsBuilder.on("progress", (status) => {
this._latestStatus = status;
return this.emit("progress", status);
});
}
async download() {
if (this._downloadStarted) {
throw new DownloadAlreadyStartedError();
}
try {
this._downloadStarted = true;
await this._engine.download();
}
finally {
await this.close();
}
}
pause() {
return this._engine.pause();
}
resume() {
return this._engine.resume();
}
close() {
return this._engine.close();
}
static async _createDownloadFile(parts, fetchStream) {
const localFileName = decodeURIComponent(new URL(parts[0], "https://example").pathname.split("/")
.pop() || "");
const downloadFile = {
totalSize: 0,
parts: [],
localFileName
};
let counter = 0;
for (const part of parts) {
try {
const { length, acceptRange, newURL, fileName } = await fetchStream.fetchDownloadInfo(part);
const downloadURL = newURL ?? part;
const size = length || 0;
downloadFile.totalSize += size;
downloadFile.parts.push({
downloadURL,
size,
acceptRange: size > 0 && acceptRange
});
if (counter++ === 0 && fileName) {
downloadFile.localFileName = fileName;
}
}
catch (error) {
if (error instanceof StatusCodeError && IGNORE_HEAD_STATUS_CODES.includes(error.statusCode)) {
// if the server does not support HEAD request, we will skip that step
downloadFile.parts.push({
downloadURL: part,
size: 0,
acceptRange: false
});
continue;
}
throw error;
}
}
return downloadFile;
}
static _validateURL(options) {
if ("partURLs" in options && "url" in options) {
throw new UrlInputError("Either `partURLs` or `url` should be provided, not both");
}
if (!("partURLs" in options) && !("url" in options)) {
throw new UrlInputError("Either `partURLs` or `url` should be provided");
}
}
static _validateOptions(options) {
if ("tryHeaders" in options && options.tryHeaders?.length && "defaultFetchDownloadInfo" in options) {
throw new InvalidOptionError("Cannot use `tryHeaders` with `defaultFetchDownloadInfo`");
}
}
}
//# sourceMappingURL=base-download-engine.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"base-download-engine.js","sourceRoot":"","sources":["../../../../src/download/download-engine/engine/base-download-engine.ts"],"names":[],"mappings":"AAGA,OAAO,aAAa,MAAM,4BAA4B,CAAC;AACvD,OAAO,EAAC,YAAY,EAAC,MAAM,eAAe,CAAC;AAC3C,OAAO,yBAAoD,MAAM,yDAAyD,CAAC;AAC3H,OAAO,2BAA2B,MAAM,2CAA2C,CAAC;AAGpF,OAAO,eAAe,MAAM,qEAAqE,CAAC;AAClG,OAAO,EAAC,kBAAkB,EAAC,MAAM,+BAA+B,CAAC;AAEjE,MAAM,wBAAwB,GAAG,CAAC,GAAG,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;AAsBjD,MAAM,CAAC,OAAO,OAAO,kBAAmB,SAAQ,YAAsC;IAClE,OAAO,CAA4B;IAChC,OAAO,CAAqB;IACrC,0BAA0B,GAAG,IAAI,yBAAyB,EAAE,CAAC;IAC7D,gBAAgB,GAAG,KAAK,CAAC;IACzB,aAAa,CAA2B;IAElD,YAAsB,MAA0B,EAAE,OAAkC;QAChF,KAAK,EAAE,CAAC;QACR,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC;QACvB,IAAI,CAAC,OAAO,GAAG,MAAM,CAAC;QACtB,IAAI,CAAC,0BAA0B,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,WAAW,EAAE,CAAC;IACvB,CAAC;IAED,IAAW,IAAI;QACX,OAAO,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;IAC7B,CAAC;IAED,IAAW,YAAY;QACnB,OAAO,IAAI,CAAC,OAAO,CAAC,YAAY,CAAC;IACrC,CAAC;IAED,IAAW,QAAQ;QACf,OAAO,IAAI,CAAC,IAAI,CAAC,aAAa,CAAC;IACnC,CAAC;IAED,IAAW,MAAM;QACb,OAAO,IAAI,CAAC,aAAa,IAAI,yBAAyB,CAAC,aAAa,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACvF,CAAC;IAED,IAAW,eAAe;QACtB,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACzB,CAAC;IAED;;OAEG;IACH,IAAW,kBAAkB;QACzB,OAAO,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC;IAChC,CAAC;IAES,WAAW;QACjB,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;YAC1B,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;QACnC,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,EAAE,GAAG,EAAE;YAC7B,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACjC,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;YAC3B,OAAO,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QAC/B,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;YAC5B,OAAO,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAChC,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,0BAA0B,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,MAAM,EAAE,EAAE;YACtD,IAAI,CAAC,aAAa,GAAG,MAAM,CAAC;YAC5B,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,EAAE,MAAM,CAAC,CAAC;QACzC,CAAC,CAAC,CAAC;IACP,CAAC;IAED,KAAK,CAAC,QAAQ;QACV,IAAI,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACxB,MAAM,IAAI,2BAA2B,EAAE,CAAC;QAC5C,CAAC;QAED,IAAI,CAAC;YACD,IAAI,CAAC,gBAAgB,GAAG,IAAI,CAAC;YAC7B,MAAM,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QAClC,CAAC;gBAAS,CAAC;YACP,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC;IACL,CAAC;IAEM,KAAK;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;IAEM,MAAM;QACT,OAAO,IAAI,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC;IACjC,CAAC;IAEM,KAAK;QACR,OAAO,IAAI,CAAC,OAAO,CAAC,KAAK,EAAE,CAAC;IAChC,CAAC;IAES,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,KAAe,EAAE,WAA0C;QAClG,MAAM,aAAa,GAAG,kBAAkB,CAAC,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,iBAAiB,CAAC,CAAC,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC;aAC5F,GAAG,EAAE,IAAI,EAAE,CAAC,CAAC;QAClB,MAAM,YAAY,GAAiB;YAC/B,SAAS,EAAE,CAAC;YACZ,KAAK,EAAE,EAAE;YACT,aAAa;SAChB,CAAC;QAEF,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACvB,IAAI,CAAC;gBACD,MAAM,EAAC,MAAM,EAAE,WAAW,EAAE,MAAM,EAAE,QAAQ,EAAC,GAAG,MAAM,WAAW,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;gBAC1F,MAAM,WAAW,GAAG,MAAM,IAAI,IAAI,CAAC;gBACnC,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,CAAC;gBAEzB,YAAY,CAAC,SAAS,IAAI,IAAI,CAAC;gBAC/B,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;oBACpB,WAAW;oBACX,IAAI;oBACJ,WAAW,EAAE,IAAI,GAAG,CAAC,IAAI,WAAW;iBACvC,CAAC,CAAC;gBAEH,IAAI,OAAO,EAAE,KAAK,CAAC,IAAI,QAAQ,EAAE,CAAC;oBAC9B,YAAY,CAAC,aAAa,GAAG,QAAQ,CAAC;gBAC1C,CAAC;YACL,CAAC;YAAC,OAAO,KAAU,EAAE,CAAC;gBAClB,IAAI,KAAK,YAAY,eAAe,IAAI,wBAAwB,CAAC,QAAQ,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC;oBAC1F,sEAAsE;oBACtE,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC;wBACpB,WAAW,EAAE,IAAI;wBACjB,IAAI,EAAE,CAAC;wBACP,WAAW,EAAE,KAAK;qBACrB,CAAC,CAAC;oBACH,SAAS;gBACb,CAAC;gBACD,MAAM,KAAK,CAAC;YAChB,CAAC;QACL,CAAC;QAED,OAAO,YAAY,CAAC;IACxB,CAAC;IAES,MAAM,CAAC,YAAY,CAAC,OAAwB;QAClD,IAAI,UAAU,IAAI,OAAO,IAAI,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5C,MAAM,IAAI,aAAa,CAAC,yDAAyD,CAAC,CAAC;QACvF,CAAC;QACD,IAAI,CAAC,CAAC,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,OAAO,CAAC,EAAE,CAAC;YAClD,MAAM,IAAI,aAAa,CAAC,+CAA+C,CAAC,CAAC;QAC7E,CAAC;IACL,CAAC;IAES,MAAM,CAAC,gBAAgB,CAAC,OAAkC;QAChE,IAAI,YAAY,IAAI,OAAO,IAAI,OAAO,CAAC,UAAU,EAAE,MAAM,IAAI,0BAA0B,IAAI,OAAO,EAAE,CAAC;YACjG,MAAM,IAAI,kBAAkB,CAAC,yDAAyD,CAAC,CAAC;QAC5F,CAAC;IACL,CAAC;CACJ"}

View File

@@ -0,0 +1,32 @@
import { SaveProgressInfo } from "../types.js";
import DownloadEngineFile from "../download-file/download-engine-file.js";
import DownloadEngineWriteStreamBrowser, { DownloadEngineWriteStreamBrowserWriter } from "../streams/download-engine-write-stream/download-engine-write-stream-browser.js";
import BaseDownloadEngine, { BaseDownloadEngineOptions } from "./base-download-engine.js";
import BaseDownloadEngineWriteStream from "../streams/download-engine-write-stream/base-download-engine-write-stream.js";
import BaseDownloadEngineFetchStream from "../streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js";
export type DownloadEngineOptionsBrowser = BaseDownloadEngineOptions & {
onWrite?: DownloadEngineWriteStreamBrowserWriter;
progress?: SaveProgressInfo;
fetchStrategy?: "xhr" | "fetch";
};
export type DownloadEngineOptionsCustomFetchBrowser = DownloadEngineOptionsBrowser & {
partURLs: string[];
fetchStream: BaseDownloadEngineFetchStream;
};
export type DownloadEngineOptionsBrowserConstructor<WriteStream = DownloadEngineWriteStreamBrowser> = DownloadEngineOptionsCustomFetchBrowser & {
writeStream: WriteStream;
};
/**
* Download engine for browser
*/
export default class DownloadEngineBrowser<WriteStream extends BaseDownloadEngineWriteStream = BaseDownloadEngineWriteStream> extends BaseDownloadEngine {
readonly options: DownloadEngineOptionsBrowserConstructor<WriteStream>;
protected constructor(engine: DownloadEngineFile, _options: DownloadEngineOptionsBrowserConstructor<WriteStream>);
get writeStream(): Omit<WriteStream, "write" | "close">;
/**
* Download file
*/
static createFromOptions(options: DownloadEngineOptionsBrowser): Promise<DownloadEngineBrowser<DownloadEngineWriteStreamBrowser>>;
protected static _createFromOptionsWithCustomFetch(options: DownloadEngineOptionsCustomFetchBrowser): Promise<DownloadEngineBrowser<DownloadEngineWriteStreamBrowser>>;
protected static _validateOptions(options: DownloadEngineOptionsBrowser): void;
}

View File

@@ -0,0 +1,52 @@
import DownloadEngineFile from "../download-file/download-engine-file.js";
import DownloadEngineFetchStreamFetch from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js";
import DownloadEngineFetchStreamXhr from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-xhr.js";
import DownloadEngineWriteStreamBrowser from "../streams/download-engine-write-stream/download-engine-write-stream-browser.js";
import BaseDownloadEngine from "./base-download-engine.js";
/**
* Download engine for browser
*/
export default class DownloadEngineBrowser extends BaseDownloadEngine {
options;
constructor(engine, _options) {
super(engine, _options);
this.options = _options;
}
get writeStream() {
return this.options.writeStream;
}
/**
* Download file
*/
static async createFromOptions(options) {
DownloadEngineBrowser._validateOptions(options);
const partURLs = "partURLs" in options ? options.partURLs : [options.url];
const fetchStream = options.fetchStrategy === "xhr" ?
new DownloadEngineFetchStreamXhr(options) : new DownloadEngineFetchStreamFetch(options);
return DownloadEngineBrowser._createFromOptionsWithCustomFetch({ ...options, partURLs, fetchStream });
}
static async _createFromOptionsWithCustomFetch(options) {
const downloadFile = await DownloadEngineBrowser._createDownloadFile(options.partURLs, options.fetchStream);
downloadFile.downloadProgress = options.progress;
const writeStream = new DownloadEngineWriteStreamBrowser(options.onWrite, {
...options,
file: downloadFile
});
if (options.acceptRangeIsKnown == null) {
const doesNotAcceptRange = downloadFile.parts.find(p => !p.acceptRange);
if (doesNotAcceptRange) {
console.warn(`Server does not accept range requests for "${doesNotAcceptRange.downloadURL}". Meaning fast-downloads/pausing/resuming will not work.
This may be related to cors origin policy (range header is ignored in the browser).
If you know the server accepts range requests, you can set "acceptRangeIsKnown" to true. To dismiss this warning, set "acceptRangeIsKnown" to false.`);
}
}
const allOptions = { ...options, writeStream };
const engine = new DownloadEngineFile(downloadFile, allOptions);
return new DownloadEngineBrowser(engine, allOptions);
}
static _validateOptions(options) {
super._validateOptions(options);
DownloadEngineBrowser._validateURL(options);
}
}
//# sourceMappingURL=download-engine-browser.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"download-engine-browser.js","sourceRoot":"","sources":["../../../../src/download/download-engine/engine/download-engine-browser.ts"],"names":[],"mappings":"AACA,OAAO,kBAAkB,MAAM,0CAA0C,CAAC;AAC1E,OAAO,8BAA8B,MAAM,+EAA+E,CAAC;AAC3H,OAAO,4BAA4B,MAAM,6EAA6E,CAAC;AACvH,OAAO,gCAA0E,MAAM,iFAAiF,CAAC;AACzK,OAAO,kBAA+C,MAAM,2BAA2B,CAAC;AAsBxF;;GAEG;AACH,MAAM,CAAC,OAAO,OAAO,qBAAyG,SAAQ,kBAAkB;IAC3H,OAAO,CAAuD;IAEvF,YAAsB,MAA0B,EAAE,QAA8D;QAC5G,KAAK,CAAC,MAAM,EAAE,QAAQ,CAAC,CAAC;QACxB,IAAI,CAAC,OAAO,GAAG,QAAQ,CAAC;IAC5B,CAAC;IAED,IAAW,WAAW;QAClB,OAAO,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC;IACpC,CAAC;IAED;;OAEG;IACI,MAAM,CAAC,KAAK,CAAC,iBAAiB,CAAC,OAAqC;QACvE,qBAAqB,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAChD,MAAM,QAAQ,GAAG,UAAU,IAAI,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC;QAE1E,MAAM,WAAW,GAAG,OAAO,CAAC,aAAa,KAAK,KAAK,CAAC,CAAC;YACjD,IAAI,4BAA4B,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,8BAA8B,CAAC,OAAO,CAAC,CAAC;QAE5F,OAAO,qBAAqB,CAAC,iCAAiC,CAAC,EAAC,GAAG,OAAO,EAAE,QAAQ,EAAE,WAAW,EAAC,CAAC,CAAC;IACxG,CAAC;IAGS,MAAM,CAAC,KAAK,CAAC,iCAAiC,CAAC,OAAgD;QACrG,MAAM,YAAY,GAAG,MAAM,qBAAqB,CAAC,mBAAmB,CAAC,OAAO,CAAC,QAAQ,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;QAC5G,YAAY,CAAC,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC;QAEjD,MAAM,WAAW,GAAG,IAAI,gCAAgC,CAAC,OAAO,CAAC,OAAO,EAAE;YACtE,GAAG,OAAO;YACV,IAAI,EAAE,YAAY;SACrB,CAAC,CAAC;QAEH,IAAI,OAAO,CAAC,kBAAkB,IAAI,IAAI,EAAE,CAAC;YACrC,MAAM,kBAAkB,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC;YACxE,IAAI,kBAAkB,EAAE,CAAC;gBACrB,OAAO,CAAC,IAAI,CAAC,8CAA8C,kBAAkB,CAAC,WAAW;;qJAE4C,CAAC,CAAC;YAC3I,CAAC;QACL,CAAC;QAED,MAAM,UAAU,GAA4C,EAAC,GAAG,OAAO,EAAE,WAAW,EAAC,CAAC;QACtF,MAAM,MAAM,GAAG,IAAI,kBAAkB,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC;QAChE,OAAO,IAAI,qBAAqB,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;IACzD,CAAC;IAES,MAAM,CAAU,gBAAgB,CAAC,OAAqC;QAC5E,KAAK,CAAC,gBAAgB,CAAC,OAAO,CAAC,CAAC;QAChC,qBAAqB,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC;IAChD,CAAC;CACJ"}

View File

@@ -0,0 +1,39 @@
import { EventEmitter } from "eventemitter3";
import { FormattedStatus } from "../../transfer-visualize/format-transfer-status.js";
import ProgressStatisticsBuilder, { ProgressStatusWithIndex } from "../../transfer-visualize/progress-statistics-builder.js";
import BaseDownloadEngine, { BaseDownloadEngineEvents } from "./base-download-engine.js";
type DownloadEngineMultiAllowedEngines = BaseDownloadEngine;
type DownloadEngineMultiDownloadEvents<Engine = DownloadEngineMultiAllowedEngines> = BaseDownloadEngineEvents & {
childDownloadStarted: (engine: Engine) => void;
childDownloadClosed: (engine: Engine) => void;
};
export type DownloadEngineMultiDownloadOptions = {
parallelDownloads?: number;
};
export default class DownloadEngineMultiDownload<Engine extends DownloadEngineMultiAllowedEngines = DownloadEngineMultiAllowedEngines> extends EventEmitter<DownloadEngineMultiDownloadEvents> {
readonly downloads: Engine[];
readonly options: DownloadEngineMultiDownloadOptions;
protected _aborted: boolean;
protected _activeEngines: Set<Engine>;
protected _progressStatisticsBuilder: ProgressStatisticsBuilder;
protected _downloadStatues: (ProgressStatusWithIndex | FormattedStatus)[];
protected _closeFiles: (() => Promise<void>)[];
protected _lastStatus?: ProgressStatusWithIndex;
protected _loadingDownloads: number;
protected constructor(engines: (DownloadEngineMultiAllowedEngines | DownloadEngineMultiDownload)[], options: DownloadEngineMultiDownloadOptions);
get downloadStatues(): (FormattedStatus | ProgressStatusWithIndex)[];
get status(): ProgressStatusWithIndex;
get downloadSize(): number;
protected _init(): void;
private _addEngine;
addDownload(engine: Engine | DownloadEngineMultiDownload<any> | Promise<Engine | DownloadEngineMultiDownload<any>>): Promise<void>;
download(): Promise<void>;
private _changeEngineFinishDownload;
private _finishEnginesDownload;
pause(): void;
resume(): void;
close(): Promise<void>;
protected static _extractEngines<Engine>(engines: Engine[]): any[];
static fromEngines<Engine extends DownloadEngineMultiAllowedEngines>(engines: (Engine | Promise<Engine>)[], options?: DownloadEngineMultiDownloadOptions): Promise<DownloadEngineMultiDownload<BaseDownloadEngine>>;
}
export {};

View File

@@ -0,0 +1,151 @@
import { EventEmitter } from "eventemitter3";
import ProgressStatisticsBuilder from "../../transfer-visualize/progress-statistics-builder.js";
import DownloadAlreadyStartedError from "./error/download-already-started-error.js";
import { concurrency } from "./utils/concurrency.js";
import { DownloadFlags, DownloadStatus } from "../download-file/progress-status-file.js";
import { NoDownloadEngineProvidedError } from "./error/no-download-engine-provided-error.js";
const DEFAULT_PARALLEL_DOWNLOADS = 1;
export default class DownloadEngineMultiDownload extends EventEmitter {
downloads;
options;
_aborted = false;
_activeEngines = new Set();
_progressStatisticsBuilder = new ProgressStatisticsBuilder();
_downloadStatues = [];
_closeFiles = [];
_lastStatus;
_loadingDownloads = 0;
constructor(engines, options) {
super();
this.downloads = DownloadEngineMultiDownload._extractEngines(engines);
this.options = options;
this._init();
}
get downloadStatues() {
return this._downloadStatues;
}
get status() {
if (!this._lastStatus) {
throw new NoDownloadEngineProvidedError();
}
return this._lastStatus;
}
get downloadSize() {
return this.downloads.reduce((acc, engine) => acc + engine.downloadSize, 0);
}
_init() {
this._progressStatisticsBuilder.downloadStatus = DownloadStatus.NotStarted;
this._progressStatisticsBuilder.on("progress", progress => {
progress = {
...progress,
downloadFlags: progress.downloadFlags.concat([DownloadFlags.DownloadSequence])
};
this._lastStatus = progress;
this.emit("progress", progress);
});
let index = 0;
for (const engine of this.downloads) {
this._addEngine(engine, index++);
}
// Prevent multiple progress events on adding engines
this._progressStatisticsBuilder.add(...this.downloads);
}
_addEngine(engine, index) {
this._downloadStatues[index] = engine.status;
engine.on("progress", (progress) => {
this._downloadStatues[index] = progress;
});
this._changeEngineFinishDownload(engine);
}
async addDownload(engine) {
const index = this.downloads.length + this._loadingDownloads;
this._downloadStatues[index] = ProgressStatisticsBuilder.loadingStatusEmptyStatistics();
this._loadingDownloads++;
this._progressStatisticsBuilder._totalDownloadParts++;
const awaitEngine = engine instanceof Promise ? await engine : engine;
this._progressStatisticsBuilder._totalDownloadParts--;
this._loadingDownloads--;
if (awaitEngine instanceof DownloadEngineMultiDownload) {
let countEngines = 0;
for (const subEngine of awaitEngine.downloads) {
this._addEngine(subEngine, index + countEngines++);
this.downloads.push(subEngine);
}
this._progressStatisticsBuilder.add(...awaitEngine.downloads);
}
else {
this._addEngine(awaitEngine, index);
this.downloads.push(awaitEngine);
this._progressStatisticsBuilder.add(awaitEngine);
}
}
async download() {
if (this._activeEngines.size) {
throw new DownloadAlreadyStartedError();
}
this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Active;
this.emit("start");
const concurrencyCount = this.options.parallelDownloads ?? DEFAULT_PARALLEL_DOWNLOADS;
await concurrency(this.downloads, concurrencyCount, async (engine) => {
if (this._aborted)
return;
this._activeEngines.add(engine);
this.emit("childDownloadStarted", engine);
await engine.download();
this.emit("childDownloadClosed", engine);
this._activeEngines.delete(engine);
});
this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Finished;
this.emit("finished");
await this._finishEnginesDownload();
await this.close();
}
_changeEngineFinishDownload(engine) {
const options = engine._fileEngineOptions;
const onFinishAsync = options.onFinishAsync;
const onCloseAsync = options.onCloseAsync;
options.onFinishAsync = undefined;
options.onCloseAsync = undefined;
this._closeFiles.push(async () => {
await onFinishAsync?.();
await options.writeStream.close();
await onCloseAsync?.();
});
}
async _finishEnginesDownload() {
await Promise.all(this._closeFiles.map(func => func()));
}
pause() {
this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Paused;
this._activeEngines.forEach(engine => engine.pause());
}
resume() {
this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Active;
this._activeEngines.forEach(engine => engine.resume());
}
async close() {
if (this._aborted)
return;
this._aborted = true;
if (this._progressStatisticsBuilder.downloadStatus !== DownloadStatus.Finished) {
this._progressStatisticsBuilder.downloadStatus = DownloadStatus.Cancelled;
}
const closePromises = Array.from(this._activeEngines)
.map(engine => engine.close());
await Promise.all(closePromises);
this.emit("closed");
}
static _extractEngines(engines) {
return engines.map(engine => {
if (engine instanceof DownloadEngineMultiDownload) {
return engine.downloads;
}
return engine;
})
.flat();
}
static async fromEngines(engines, options = {}) {
return new DownloadEngineMultiDownload(await Promise.all(engines), options);
}
}
//# sourceMappingURL=download-engine-multi-download.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,67 @@
import { DownloadFile } from "../types.js";
import DownloadEngineFile from "../download-file/download-engine-file.js";
import DownloadEngineWriteStreamNodejs from "../streams/download-engine-write-stream/download-engine-write-stream-nodejs.js";
import BaseDownloadEngine, { BaseDownloadEngineOptions } from "./base-download-engine.js";
import BaseDownloadEngineFetchStream from "../streams/download-engine-fetch-stream/base-download-engine-fetch-stream.js";
export declare const PROGRESS_FILE_EXTENSION = ".ipull";
type PathOptions = {
directory: string;
} | {
savePath: string;
};
export type DownloadEngineOptionsNodejs = PathOptions & BaseDownloadEngineOptions & {
fileName?: string;
fetchStrategy?: "localFile" | "fetch";
skipExisting?: boolean;
};
export type DownloadEngineOptionsNodejsCustomFetch = DownloadEngineOptionsNodejs & {
partURLs: string[];
fetchStream: BaseDownloadEngineFetchStream;
};
export type DownloadEngineOptionsNodejsConstructor<WriteStream = DownloadEngineWriteStreamNodejs> = DownloadEngineOptionsNodejsCustomFetch & {
writeStream: WriteStream;
};
/**
* Download engine for Node.js
*/
export default class DownloadEngineNodejs<T extends DownloadEngineWriteStreamNodejs = DownloadEngineWriteStreamNodejs> extends BaseDownloadEngine {
readonly options: DownloadEngineOptionsNodejsConstructor<T>;
protected constructor(engine: DownloadEngineFile, _options: DownloadEngineOptionsNodejsConstructor<T>);
protected _initEvents(): void;
/**
* The file path with the progress extension or the final file path if the download is finished
*/
get fileAbsolutePath(): string;
/**
* The final file path (without the progress extension)
*/
get finalFileAbsolutePath(): string;
/**
* Abort the download & delete the file (**even if** the download is finished)
* @deprecated use `close` with flag `deleteFile` instead
*
* TODO: remove in the next major version
*/
closeAndDeleteFile(): Promise<void>;
/**
* Close the download engine
* @param deleteTempFile {boolean} - delete the temp file (when the download is **not finished**).
* @param deleteFile {boolean} - delete the **temp** or **final file** (clean everything up).
*/
close({ deleteTempFile, deleteFile }?: {
deleteTempFile?: boolean;
deleteFile?: boolean;
}): Promise<void>;
/**
* Download/copy a file
*
* if `fetchStrategy` is defined as "localFile" it will copy the file, otherwise it will download it
* By default, it will guess the strategy based on the URL
*/
static createFromOptions(options: DownloadEngineOptionsNodejs): Promise<DownloadEngineNodejs<DownloadEngineWriteStreamNodejs>>;
protected static _createFromOptionsWithCustomFetch(options: DownloadEngineOptionsNodejsCustomFetch): Promise<DownloadEngineNodejs<DownloadEngineWriteStreamNodejs>>;
protected static _createDownloadLocation(download: DownloadFile, options: DownloadEngineOptionsNodejs): string;
protected static _validateOptions(options: DownloadEngineOptionsNodejs): void;
protected static _guessFetchStrategy(url: string): "localFile" | "fetch";
}
export {};

View File

@@ -0,0 +1,154 @@
import path from "path";
import DownloadEngineFile from "../download-file/download-engine-file.js";
import DownloadEngineFetchStreamFetch from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.js";
import DownloadEngineWriteStreamNodejs from "../streams/download-engine-write-stream/download-engine-write-stream-nodejs.js";
import DownloadEngineFetchStreamLocalFile from "../streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.js";
import BaseDownloadEngine from "./base-download-engine.js";
import SavePathError from "./error/save-path-error.js";
import fs from "fs-extra";
import filenamify from "filenamify";
import { DownloadStatus } from "../download-file/progress-status-file.js";
export const PROGRESS_FILE_EXTENSION = ".ipull";
/**
* Download engine for Node.js
*/
export default class DownloadEngineNodejs extends BaseDownloadEngine {
options;
constructor(engine, _options) {
super(engine, _options);
this.options = _options;
}
_initEvents() {
super._initEvents();
this._engine.options.onSaveProgressAsync = async (progress) => {
if (this.options.skipExisting)
return;
await this.options.writeStream.saveMedataAfterFile(progress);
};
// Try to clone the file if it's a single part download
this._engine.options.onStartedAsync = async () => {
if (this.options.skipExisting || this.options.fetchStrategy !== "localFile" || this.options.partURLs.length !== 1)
return;
try {
const { reflinkFile } = await import("@reflink/reflink");
await fs.remove(this.options.writeStream.path);
await reflinkFile(this.options.partURLs[0], this.options.writeStream.path);
this._engine.finished("cloned");
}
catch { }
};
this._engine.options.onFinishAsync = async () => {
if (this.options.skipExisting)
return;
await this.options.writeStream.ftruncate(this.downloadSize);
};
this._engine.options.onCloseAsync = async () => {
if (this.status.ended && this.options.writeStream.path != this.options.writeStream.finalPath) {
await fs.rename(this.options.writeStream.path, this.options.writeStream.finalPath);
this.options.writeStream.path = this.options.writeStream.finalPath;
}
};
if (this.options.skipExisting) {
this.options.writeStream.path = this.options.writeStream.finalPath;
}
}
/**
* The file path with the progress extension or the final file path if the download is finished
*/
get fileAbsolutePath() {
return path.resolve(this.options.writeStream.path);
}
/**
* The final file path (without the progress extension)
*/
get finalFileAbsolutePath() {
return path.resolve(this.options.writeStream.finalPath);
}
/**
* Abort the download & delete the file (**even if** the download is finished)
* @deprecated use `close` with flag `deleteFile` instead
*
* TODO: remove in the next major version
*/
async closeAndDeleteFile() {
await this.close({ deleteFile: true });
}
/**
* Close the download engine
* @param deleteTempFile {boolean} - delete the temp file (when the download is **not finished**).
* @param deleteFile {boolean} - delete the **temp** or **final file** (clean everything up).
*/
async close({ deleteTempFile, deleteFile } = {}) {
await super.close();
if (deleteFile || deleteTempFile && this.status.downloadStatus != DownloadStatus.Finished) {
try {
await fs.unlink(this.fileAbsolutePath);
}
catch { }
}
}
/**
* Download/copy a file
*
* if `fetchStrategy` is defined as "localFile" it will copy the file, otherwise it will download it
* By default, it will guess the strategy based on the URL
*/
static async createFromOptions(options) {
DownloadEngineNodejs._validateOptions(options);
const partURLs = "partURLs" in options ? options.partURLs : [options.url];
options.fetchStrategy ??= DownloadEngineNodejs._guessFetchStrategy(partURLs[0]);
const fetchStream = options.fetchStrategy === "localFile" ?
new DownloadEngineFetchStreamLocalFile(options) :
new DownloadEngineFetchStreamFetch(options);
return DownloadEngineNodejs._createFromOptionsWithCustomFetch({ ...options, partURLs, fetchStream });
}
static async _createFromOptionsWithCustomFetch(options) {
const downloadFile = await DownloadEngineNodejs._createDownloadFile(options.partURLs, options.fetchStream);
const downloadLocation = DownloadEngineNodejs._createDownloadLocation(downloadFile, options);
downloadFile.localFileName = path.basename(downloadLocation);
const writeStream = new DownloadEngineWriteStreamNodejs(downloadLocation + PROGRESS_FILE_EXTENSION, downloadLocation, options);
writeStream.fileSize = downloadFile.totalSize;
downloadFile.downloadProgress = await writeStream.loadMetadataAfterFileWithoutRetry();
if (options.skipExisting) {
options.skipExisting = false;
if (downloadFile.totalSize > 0 && !downloadFile.downloadProgress) {
try {
const stat = await fs.stat(downloadLocation);
if (stat.isFile() && stat.size === downloadFile.totalSize) {
options.skipExisting = true;
}
}
catch { }
}
}
const allOptions = { ...options, writeStream };
const engine = new DownloadEngineFile(downloadFile, allOptions);
return new DownloadEngineNodejs(engine, allOptions);
}
static _createDownloadLocation(download, options) {
if ("savePath" in options) {
return options.savePath;
}
const fileName = options.fileName || download.localFileName;
return path.join(options.directory, filenamify(fileName));
}
static _validateOptions(options) {
super._validateOptions(options);
if (!("directory" in options) && !("savePath" in options)) {
throw new SavePathError("Either `directory` or `savePath` must be provided");
}
if ("directory" in options && "savePath" in options) {
throw new SavePathError("Both `directory` and `savePath` cannot be provided");
}
DownloadEngineNodejs._validateURL(options);
}
static _guessFetchStrategy(url) {
try {
new URL(url);
return "fetch";
}
catch { }
return "localFile";
}
}
//# sourceMappingURL=download-engine-nodejs.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,4 @@
import EngineError from "./engine-error.js";
export declare class InvalidOptionError extends EngineError {
constructor(message: string);
}

View File

@@ -0,0 +1,7 @@
import EngineError from "./engine-error.js";
export class InvalidOptionError extends EngineError {
constructor(message) {
super(message);
}
}
//# sourceMappingURL=InvalidOptionError.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"InvalidOptionError.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/engine/error/InvalidOptionError.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,mBAAmB,CAAC;AAE5C,MAAM,OAAO,kBAAmB,SAAQ,WAAW;IAC/C,YAAY,OAAe;QACvB,KAAK,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;CACJ"}

View File

@@ -0,0 +1,4 @@
import EngineError from "./engine-error.js";
export default class DownloadAlreadyStartedError extends EngineError {
constructor();
}

View File

@@ -0,0 +1,7 @@
import EngineError from "./engine-error.js";
export default class DownloadAlreadyStartedError extends EngineError {
constructor() {
super("Download already started");
}
}
//# sourceMappingURL=download-already-started-error.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"download-already-started-error.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/engine/error/download-already-started-error.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,mBAAmB,CAAC;AAE5C,MAAM,CAAC,OAAO,OAAO,2BAA4B,SAAQ,WAAW;IAChE;QACI,KAAK,CAAC,0BAA0B,CAAC,CAAC;IACtC,CAAC;CACJ"}

View File

@@ -0,0 +1,3 @@
import IpullError from "../../../../errors/ipull-error.js";
export default class EngineError extends IpullError {
}

View File

@@ -0,0 +1,4 @@
import IpullError from "../../../../errors/ipull-error.js";
export default class EngineError extends IpullError {
}
//# sourceMappingURL=engine-error.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"engine-error.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/engine/error/engine-error.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,mCAAmC,CAAC;AAE3D,MAAM,CAAC,OAAO,OAAO,WAAY,SAAQ,UAAU;CAAG"}

View File

@@ -0,0 +1,4 @@
import EngineError from "./engine-error.js";
export default class InvalidContentLengthError extends EngineError {
constructor(url: string);
}

View File

@@ -0,0 +1,7 @@
import EngineError from "./engine-error.js";
export default class InvalidContentLengthError extends EngineError {
constructor(url) {
super(`Invalid content length, for request URL: ${url}`);
}
}
//# sourceMappingURL=invalid-content-length-error.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"invalid-content-length-error.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/engine/error/invalid-content-length-error.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,mBAAmB,CAAC;AAE5C,MAAM,CAAC,OAAO,OAAO,yBAA0B,SAAQ,WAAW;IAC9D,YAAY,GAAW;QACnB,KAAK,CAAC,4CAA4C,GAAG,EAAE,CAAC,CAAC;IAC7D,CAAC;CACJ"}

View File

@@ -0,0 +1,3 @@
import EngineError from "./engine-error.js";
export default class MissingEventError extends EngineError {
}

View File

@@ -0,0 +1,4 @@
import EngineError from "./engine-error.js";
export default class MissingEventError extends EngineError {
}
//# sourceMappingURL=missing-event-error.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"missing-event-error.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/engine/error/missing-event-error.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,mBAAmB,CAAC;AAE5C,MAAM,CAAC,OAAO,OAAO,iBAAkB,SAAQ,WAAW;CAAG"}

View File

@@ -0,0 +1,4 @@
import EngineError from "./engine-error.js";
export declare class NoDownloadEngineProvidedError extends EngineError {
constructor(error?: string);
}

View File

@@ -0,0 +1,7 @@
import EngineError from "./engine-error.js";
export class NoDownloadEngineProvidedError extends EngineError {
constructor(error = "No download engine provided for download sequence") {
super(error);
}
}
//# sourceMappingURL=no-download-engine-provided-error.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"no-download-engine-provided-error.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/engine/error/no-download-engine-provided-error.ts"],"names":[],"mappings":"AAAA,OAAO,WAAW,MAAM,mBAAmB,CAAC;AAE5C,MAAM,OAAO,6BAA8B,SAAQ,WAAW;IAC1D,YAAY,KAAK,GAAG,mDAAmD;QACnE,KAAK,CAAC,KAAK,CAAC,CAAC;IACjB,CAAC;CACJ"}

View File

@@ -0,0 +1,4 @@
import { InvalidOptionError } from "./InvalidOptionError.js";
export default class SavePathError extends InvalidOptionError {
constructor(message: string);
}

View File

@@ -0,0 +1,7 @@
import { InvalidOptionError } from "./InvalidOptionError.js";
export default class SavePathError extends InvalidOptionError {
constructor(message) {
super(message);
}
}
//# sourceMappingURL=save-path-error.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"save-path-error.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/engine/error/save-path-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,kBAAkB,EAAC,MAAM,yBAAyB,CAAC;AAE3D,MAAM,CAAC,OAAO,OAAO,aAAc,SAAQ,kBAAkB;IACzD,YAAY,OAAe;QACvB,KAAK,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;CACJ"}

View File

@@ -0,0 +1,4 @@
import { InvalidOptionError } from "./InvalidOptionError.js";
export default class UrlInputError extends InvalidOptionError {
constructor(message: string);
}

View File

@@ -0,0 +1,7 @@
import { InvalidOptionError } from "./InvalidOptionError.js";
export default class UrlInputError extends InvalidOptionError {
constructor(message) {
super(message);
}
}
//# sourceMappingURL=url-input-error.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"url-input-error.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/engine/error/url-input-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,kBAAkB,EAAC,MAAM,yBAAyB,CAAC;AAE3D,MAAM,CAAC,OAAO,OAAO,aAAc,SAAQ,kBAAkB;IACzD,YAAY,OAAe;QACvB,KAAK,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;CACJ"}

View File

@@ -0,0 +1 @@
export declare function concurrency<Value>(array: Value[], concurrencyCount: number, callback: (value: Value) => Promise<void>): Promise<void>;

View File

@@ -0,0 +1,22 @@
export function concurrency(array, concurrencyCount, callback) {
return new Promise((resolve, reject) => {
let index = 0;
let activeCount = 0;
function next() {
if (index === array.length && activeCount === 0) {
resolve();
return;
}
while (activeCount < concurrencyCount && index < array.length) {
activeCount++;
callback(array[index++])
.then(() => {
activeCount--;
next();
}, reject);
}
}
next();
});
}
//# sourceMappingURL=concurrency.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"concurrency.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/engine/utils/concurrency.ts"],"names":[],"mappings":"AAAA,MAAM,UAAU,WAAW,CAAQ,KAAc,EAAE,gBAAwB,EAAE,QAAyC;IAClH,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACnC,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,WAAW,GAAG,CAAC,CAAC;QAEpB,SAAS,IAAI;YACT,IAAI,KAAK,KAAK,KAAK,CAAC,MAAM,IAAI,WAAW,KAAK,CAAC,EAAE,CAAC;gBAC9C,OAAO,EAAE,CAAC;gBACV,OAAO;YACX,CAAC;YAED,OAAO,WAAW,GAAG,gBAAgB,IAAI,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC;gBAC5D,WAAW,EAAE,CAAC;gBACd,QAAQ,CAAC,KAAK,CAAC,KAAK,EAAE,CAAC,CAAC;qBACnB,IAAI,CAAC,GAAG,EAAE;oBACP,WAAW,EAAE,CAAC;oBACd,IAAI,EAAE,CAAC;gBACX,CAAC,EAAE,MAAM,CAAC,CAAC;YACnB,CAAC;QACL,CAAC;QAED,IAAI,EAAE,CAAC;IACX,CAAC,CAAC,CAAC;AACP,CAAC"}

View File

@@ -0,0 +1,79 @@
import retry from "async-retry";
import { EventEmitter } from "eventemitter3";
import { AvailablePrograms } from "../../download-file/download-programs/switch-program.js";
import StatusCodeError from "./errors/status-code-error.js";
export declare const MIN_LENGTH_FOR_MORE_INFO_REQUEST: number;
export type BaseDownloadEngineFetchStreamOptions = {
retry?: retry.Options;
/**
* If true, the engine will retry the request if the server returns a status code between 500 and 599
*/
retryOnServerError?: boolean;
headers?: Record<string, string>;
/**
* If true, parallel download will be enabled even if the server does not return `accept-range` header, this is good when using cross-origin requests
*/
acceptRangeIsKnown?: boolean;
ignoreIfRangeWithQueryParams?: boolean;
} & ({
defaultFetchDownloadInfo?: {
length: number;
acceptRange: boolean;
};
} | {
/**
* Try different headers to see if any authentication is needed
*/
tryHeaders?: Record<string, string>[];
/**
* Delay between trying different headers
*/
tryHeadersDelay?: number;
});
export type DownloadInfoResponse = {
length: number;
acceptRange: boolean;
newURL?: string;
fileName?: string;
};
export type FetchSubState = {
url: string;
startChunk: number;
endChunk: number;
totalSize: number;
chunkSize: number;
rangeSupport?: boolean;
onProgress?: (length: number) => void;
};
export type BaseDownloadEngineFetchStreamEvents = {
paused: () => void;
resumed: () => void;
aborted: () => void;
errorCountIncreased: (errorCount: number, error: Error) => void;
};
export type WriteCallback = (data: Uint8Array[], position: number, index: number) => void;
export default abstract class BaseDownloadEngineFetchStream extends EventEmitter<BaseDownloadEngineFetchStreamEvents> {
readonly programType?: AvailablePrograms;
abstract readonly transferAction: string;
readonly options: Partial<BaseDownloadEngineFetchStreamOptions>;
state: FetchSubState;
paused?: Promise<void>;
aborted: boolean;
protected _pausedResolve?: () => void;
errorCount: {
value: number;
};
constructor(options?: Partial<BaseDownloadEngineFetchStreamOptions>);
protected get _startSize(): number;
protected get _endSize(): number;
protected initEvents(): void;
abstract withSubState(state: FetchSubState): this;
protected cloneState<Fetcher extends BaseDownloadEngineFetchStream>(state: FetchSubState, fetchStream: Fetcher): Fetcher;
fetchDownloadInfo(url: string): Promise<DownloadInfoResponse>;
protected abstract fetchDownloadInfoWithoutRetry(url: string): Promise<DownloadInfoResponse>;
fetchChunks(callback: WriteCallback): Promise<void>;
protected abstract fetchWithoutRetryChunks(callback: WriteCallback): Promise<void> | void;
close(): void | Promise<void>;
protected appendToURL(url: string): string;
protected retryOnServerError(error: Error): error is StatusCodeError;
}

View File

@@ -0,0 +1,139 @@
import retry from "async-retry";
import { retryAsyncStatementSimple } from "./utils/retry-async-statement.js";
import { EventEmitter } from "eventemitter3";
import HttpError from "./errors/http-error.js";
import StatusCodeError from "./errors/status-code-error.js";
import sleep from "sleep-promise";
export const MIN_LENGTH_FOR_MORE_INFO_REQUEST = 1024 * 1024 * 3; // 3MB
const DEFAULT_OPTIONS = {
retryOnServerError: true,
retry: {
retries: 150,
factor: 1.5,
minTimeout: 200,
maxTimeout: 5_000
},
tryHeadersDelay: 50
};
export default class BaseDownloadEngineFetchStream extends EventEmitter {
programType;
options = {};
state = null;
paused;
aborted = false;
_pausedResolve;
errorCount = { value: 0 };
constructor(options = {}) {
super();
this.options = { ...DEFAULT_OPTIONS, ...options };
this.initEvents();
}
get _startSize() {
return this.state.startChunk * this.state.chunkSize;
}
get _endSize() {
return Math.min(this.state.endChunk * this.state.chunkSize, this.state.totalSize);
}
initEvents() {
this.on("aborted", () => {
this.aborted = true;
this._pausedResolve?.();
});
this.on("paused", () => {
this.paused = new Promise((resolve) => {
this._pausedResolve = resolve;
});
});
this.on("resumed", () => {
this._pausedResolve?.();
this._pausedResolve = undefined;
this.paused = undefined;
});
}
cloneState(state, fetchStream) {
fetchStream.state = state;
fetchStream.errorCount = this.errorCount;
fetchStream.on("errorCountIncreased", this.emit.bind(this, "errorCountIncreased"));
this.on("aborted", fetchStream.emit.bind(fetchStream, "aborted"));
this.on("paused", fetchStream.emit.bind(fetchStream, "paused"));
this.on("resumed", fetchStream.emit.bind(fetchStream, "resumed"));
return fetchStream;
}
async fetchDownloadInfo(url) {
let throwErr = null;
const tryHeaders = "tryHeaders" in this.options && this.options.tryHeaders ? this.options.tryHeaders.slice() : [];
const fetchDownloadInfoCallback = async () => {
try {
return await this.fetchDownloadInfoWithoutRetry(url);
}
catch (error) {
if (error instanceof HttpError && !this.retryOnServerError(error)) {
if ("tryHeaders" in this.options && tryHeaders.length) {
this.options.headers = tryHeaders.shift();
await sleep(this.options.tryHeadersDelay ?? 0);
return await fetchDownloadInfoCallback();
}
throwErr = error;
return null;
}
this.errorCount.value++;
this.emit("errorCountIncreased", this.errorCount.value, error);
if (error instanceof StatusCodeError && error.retryAfter) {
await sleep(error.retryAfter * 1000);
return await fetchDownloadInfoCallback();
}
throw error;
}
};
const response = ("defaultFetchDownloadInfo" in this.options && this.options.defaultFetchDownloadInfo) || await retry(fetchDownloadInfoCallback, this.options.retry);
if (throwErr) {
throw throwErr;
}
return response;
}
async fetchChunks(callback) {
let lastStartLocation = this.state.startChunk;
let retryResolvers = retryAsyncStatementSimple(this.options.retry);
// eslint-disable-next-line no-constant-condition
while (true) {
try {
return await this.fetchWithoutRetryChunks(callback);
}
catch (error) {
if (error?.name === "AbortError")
return;
if (error instanceof HttpError && !this.retryOnServerError(error)) {
throw error;
}
this.errorCount.value++;
this.emit("errorCountIncreased", this.errorCount.value, error);
if (error instanceof StatusCodeError && error.retryAfter) {
await sleep(error.retryAfter * 1000);
continue;
}
if (lastStartLocation !== this.state.startChunk) {
lastStartLocation = this.state.startChunk;
retryResolvers = retryAsyncStatementSimple(this.options.retry);
}
await retryResolvers(error);
}
}
}
close() {
this.emit("aborted");
}
appendToURL(url) {
const parsed = new URL(url);
if (this.options.ignoreIfRangeWithQueryParams) {
const randomText = Math.random()
.toString(36);
parsed.searchParams.set("_ignore", randomText);
}
return parsed.href;
}
retryOnServerError(error) {
return Boolean(this.options.retryOnServerError) && error instanceof StatusCodeError &&
(error.statusCode >= 500 || error.statusCode === 429);
}
}
//# sourceMappingURL=base-download-engine-fetch-stream.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,16 @@
import BaseDownloadEngineFetchStream, { DownloadInfoResponse, FetchSubState, WriteCallback } from "./base-download-engine-fetch-stream.js";
type GetNextChunk = () => Promise<ReadableStreamReadResult<Uint8Array>> | ReadableStreamReadResult<Uint8Array>;
export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFetchStream {
private _fetchDownloadInfoWithHEAD;
transferAction: string;
withSubState(state: FetchSubState): this;
protected fetchWithoutRetryChunks(callback: WriteCallback): Promise<void>;
protected fetchDownloadInfoWithoutRetry(url: string): Promise<DownloadInfoResponse>;
protected fetchDownloadInfoWithoutRetryByMethod(url: string, method?: "HEAD" | "GET"): Promise<DownloadInfoResponse>;
protected fetchDownloadInfoWithoutRetryContentRange(url: string, response?: Response): Promise<number>;
chunkGenerator(callback: WriteCallback, getNextChunk: GetNextChunk): Promise<void>;
protected static convertHeadersToRecord(headers: Headers): {
[key: string]: string;
};
}
export {};

View File

@@ -0,0 +1,113 @@
import BaseDownloadEngineFetchStream, { MIN_LENGTH_FOR_MORE_INFO_REQUEST } from "./base-download-engine-fetch-stream.js";
import InvalidContentLengthError from "./errors/invalid-content-length-error.js";
import SmartChunkSplit from "./utils/smart-chunk-split.js";
import { parseContentDisposition } from "./utils/content-disposition.js";
import StatusCodeError from "./errors/status-code-error.js";
import { parseHttpContentRange } from "./utils/httpRange.js";
import { browserCheck } from "./utils/browserCheck.js";
export default class DownloadEngineFetchStreamFetch extends BaseDownloadEngineFetchStream {
_fetchDownloadInfoWithHEAD = false;
transferAction = "Downloading";
withSubState(state) {
const fetchStream = new DownloadEngineFetchStreamFetch(this.options);
return this.cloneState(state, fetchStream);
}
async fetchWithoutRetryChunks(callback) {
const headers = {
accept: "*/*",
...this.options.headers
};
if (this.state.rangeSupport) {
headers.range = `bytes=${this._startSize}-${this._endSize - 1}`;
}
const controller = new AbortController();
const response = await fetch(this.appendToURL(this.state.url), {
headers,
signal: controller.signal
});
if (response.status < 200 || response.status >= 300) {
throw new StatusCodeError(this.state.url, response.status, response.statusText, headers);
}
const contentLength = parseHttpContentRange(response.headers.get("content-range"))?.length ?? parseInt(response.headers.get("content-length"));
const expectedContentLength = this._endSize - this._startSize;
if (this.state.rangeSupport && contentLength !== expectedContentLength) {
throw new InvalidContentLengthError(expectedContentLength, contentLength);
}
this.on("aborted", () => {
controller.abort();
});
const reader = response.body.getReader();
return await this.chunkGenerator(callback, () => reader.read());
}
async fetchDownloadInfoWithoutRetry(url) {
if (this._fetchDownloadInfoWithHEAD) {
try {
return this.fetchDownloadInfoWithoutRetryByMethod(url, "HEAD");
}
catch (error) {
if (!(error instanceof StatusCodeError)) {
throw error;
}
this._fetchDownloadInfoWithHEAD = false;
}
}
return this.fetchDownloadInfoWithoutRetryByMethod(url, "GET");
}
async fetchDownloadInfoWithoutRetryByMethod(url, method = "HEAD") {
const response = await fetch(url, {
method: method,
headers: {
"Accept-Encoding": "identity",
...this.options.headers
}
});
if (response.status < 200 || response.status >= 300) {
throw new StatusCodeError(url, response.status, response.statusText, this.options.headers, DownloadEngineFetchStreamFetch.convertHeadersToRecord(response.headers));
}
const acceptRange = this.options.acceptRangeIsKnown ?? response.headers.get("accept-ranges") === "bytes";
const fileName = parseContentDisposition(response.headers.get("content-disposition"));
let length = parseInt(response.headers.get("content-length")) || 0;
if (response.headers.get("content-encoding") || browserCheck() && MIN_LENGTH_FOR_MORE_INFO_REQUEST < length) {
length = acceptRange ? await this.fetchDownloadInfoWithoutRetryContentRange(url, method === "GET" ? response : undefined) : 0;
}
return {
length,
acceptRange,
newURL: response.url,
fileName
};
}
async fetchDownloadInfoWithoutRetryContentRange(url, response) {
const responseGet = response ?? await fetch(url, {
method: "GET",
headers: {
accept: "*/*",
...this.options.headers,
range: "bytes=0-0"
}
});
const contentRange = responseGet.headers.get("content-range");
return parseHttpContentRange(contentRange)?.size || 0;
}
async chunkGenerator(callback, getNextChunk) {
const smartSplit = new SmartChunkSplit(callback, this.state);
// eslint-disable-next-line no-constant-condition
while (true) {
const { done, value } = await getNextChunk();
await this.paused;
if (done || this.aborted)
break;
smartSplit.addChunk(value);
this.state.onProgress?.(smartSplit.savedLength);
}
smartSplit.sendLeftovers();
}
static convertHeadersToRecord(headers) {
const headerObj = {};
headers.forEach((value, key) => {
headerObj[key] = value;
});
return headerObj;
}
}
//# sourceMappingURL=download-engine-fetch-stream-fetch.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"download-engine-fetch-stream-fetch.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-fetch.ts"],"names":[],"mappings":"AAAA,OAAO,6BAA6B,EAAE,EAAsC,gCAAgC,EAAgB,MAAM,wCAAwC,CAAC;AAC3K,OAAO,yBAAyB,MAAM,0CAA0C,CAAC;AACjF,OAAO,eAAe,MAAM,8BAA8B,CAAC;AAC3D,OAAO,EAAC,uBAAuB,EAAC,MAAM,gCAAgC,CAAC;AACvE,OAAO,eAAe,MAAM,+BAA+B,CAAC;AAC5D,OAAO,EAAC,qBAAqB,EAAC,MAAM,sBAAsB,CAAC;AAC3D,OAAO,EAAC,YAAY,EAAC,MAAM,yBAAyB,CAAC;AAGrD,MAAM,CAAC,OAAO,OAAO,8BAA+B,SAAQ,6BAA6B;IAC7E,0BAA0B,GAAG,KAAK,CAAC;IAC3B,cAAc,GAAG,aAAa,CAAC;IAE/C,YAAY,CAAC,KAAoB;QAC7B,MAAM,WAAW,GAAG,IAAI,8BAA8B,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACrE,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,CAAS,CAAC;IACvD,CAAC;IAEkB,KAAK,CAAC,uBAAuB,CAAC,QAAuB;QACpE,MAAM,OAAO,GAA2B;YACpC,MAAM,EAAE,KAAK;YACb,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO;SAC1B,CAAC;QAEF,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,EAAE,CAAC;YAC1B,OAAO,CAAC,KAAK,GAAG,SAAS,IAAI,CAAC,UAAU,IAAI,IAAI,CAAC,QAAQ,GAAG,CAAC,EAAE,CAAC;QACpE,CAAC;QAED,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE;YAC3D,OAAO;YACP,MAAM,EAAE,UAAU,CAAC,MAAM;SAC5B,CAAC,CAAC;QAEH,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAClD,MAAM,IAAI,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,CAAC;QAC7F,CAAC;QAED,MAAM,aAAa,GAAG,qBAAqB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC,EAAE,MAAM,IAAI,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAE,CAAC,CAAC;QAChJ,MAAM,qBAAqB,GAAG,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC;QAC9D,IAAI,IAAI,CAAC,KAAK,CAAC,YAAY,IAAI,aAAa,KAAK,qBAAqB,EAAE,CAAC;YACrE,MAAM,IAAI,yBAAyB,CAAC,qBAAqB,EAAE,aAAa,CAAC,CAAC;QAC9E,CAAC;QAED,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,GAAG,EAAE;YACpB,UAAU,CAAC,KAAK,EAAE,CAAC;QACvB,CAAC,CAAC,CAAC;QAEH,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAK,CAAC,SAAS,EAAE,CAAC;QAC1C,OAAO,MAAM,IAAI,CAAC,cAAc,CAAC,QAAQ,EAAE,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC;IACpE,CAAC;IAEkB,KAAK,CAAC,6BAA6B,CAAC,GAAW;QAC9D,IAAI,IAAI,CAAC,0BAA0B,EAAE,CAAC;YAClC,IAAI,CAAC;gBACD,OAAO,IAAI,CAAC,qCAAqC,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACnE,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACb,IAAI,CAAC,CAAC,KAAK,YAAY,eAAe,CAAC,EAAE,CAAC;oBACtC,MAAM,KAAK,CAAC;gBAChB,CAAC;gBACD,IAAI,CAAC,0BAA0B,GAAG,KAAK,CAAC;YAC5C,CAAC;QACL,CAAC;QAED,OAAO,IAAI,CAAC,qCAAqC,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;IAClE,CAAC;IAES,KAAK,CAAC,qCAAqC,CAAC,GAAW,EAAE,SAAyB,MAAM;QAC9F,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;YAC9B,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACL,iBAAiB,EAAE,UAAU;gBAC7B,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO;aAC1B;SACJ,CAAC,CAAC;QAGH,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAClD,MAAM,IAAI,eAAe,CAAC,GAAG,EAAE,QAAQ,CAAC,MAAM,EAAE,QAAQ,CAAC,UAAU,EAAE,IAAI,CAAC,OAAO,CAAC,OAAO,EAAE,8BAA8B,CAAC,sBAAsB,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC;QACxK,CAAC;QAED,MAAM,WAAW,GAAG,IAAI,CAAC,OAAO,CAAC,kBAAkB,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,KAAK,OAAO,CAAC;QACzG,MAAM,QAAQ,GAAG,uBAAuB,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC,CAAC;QAEtF,IAAI,MAAM,GAAG,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAE,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,IAAI,YAAY,EAAE,IAAI,gCAAgC,GAAG,MAAM,EAAE,CAAC;YAC1G,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,MAAM,IAAI,CAAC,yCAAyC,CAAC,GAAG,EAAE,MAAM,KAAK,KAAK,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QAClI,CAAC;QAED,OAAO;YACH,MAAM;YACN,WAAW;YACX,MAAM,EAAE,QAAQ,CAAC,GAAG;YACpB,QAAQ;SACX,CAAC;IACN,CAAC;IAES,KAAK,CAAC,yCAAyC,CAAC,GAAW,EAAE,QAAmB;QACtF,MAAM,WAAW,GAAG,QAAQ,IAAI,MAAM,KAAK,CAAC,GAAG,EAAE;YAC7C,MAAM,EAAE,KAAK;YACb,OAAO,EAAE;gBACL,MAAM,EAAE,KAAK;gBACb,GAAG,IAAI,CAAC,OAAO,CAAC,OAAO;gBACvB,KAAK,EAAE,WAAW;aACrB;SACJ,CAAC,CAAC;QAEH,MAAM,YAAY,GAAG,WAAW,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,CAAC;QAC9D,OAAO,qBAAqB,CAAC,YAAY,CAAC,EAAE,IAAI,IAAI,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,CAAC,cAAc,CAAC,QAAuB,EAAE,YAA0B;QACpE,MAAM,UAAU,GAAG,IAAI,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC;QAE7D,iDAAiD;QACjD,OAAO,IAAI,EAAE,CAAC;YACV,MAAM,EAAC,IAAI,EAAE,KAAK,EAAC,GAAG,MAAM,YAAY,EAAE,CAAC;YAC3C,MAAM,IAAI,CAAC,MAAM,CAAC;YAClB,IAAI,IAAI,IAAI,IAAI,CAAC,OAAO;gBAAE,MAAM;YAEhC,UAAU,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC;YAC3B,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QACpD,CAAC;QAED,UAAU,CAAC,aAAa,EAAE,CAAC;IAC/B,CAAC;IAES,MAAM,CAAC,sBAAsB,CAAC,OAAgB;QACpD,MAAM,SAAS,GAA8B,EAAE,CAAC;QAChD,OAAO,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,EAAE;YAC3B,SAAS,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC3B,CAAC,CAAC,CAAC;QACH,OAAO,SAAS,CAAC;IACrB,CAAC;CACJ"}

View File

@@ -0,0 +1,11 @@
import BaseDownloadEngineFetchStream, { DownloadInfoResponse, FetchSubState, WriteCallback } from "./base-download-engine-fetch-stream.js";
export default class DownloadEngineFetchStreamLocalFile extends BaseDownloadEngineFetchStream {
transferAction: string;
private _fd;
private _fsPath;
withSubState(state: FetchSubState): this;
private _ensureFileOpen;
protected fetchWithoutRetryChunks(callback: WriteCallback): Promise<void>;
protected fetchDownloadInfoWithoutRetry(path: string): Promise<DownloadInfoResponse>;
close(): void;
}

View File

@@ -0,0 +1,54 @@
import fs from "fs/promises";
import { withLock } from "lifecycle-utils";
import retry from "async-retry";
import fsExtra from "fs-extra";
import BaseDownloadEngineFetchStream from "./base-download-engine-fetch-stream.js";
import SmartChunkSplit from "./utils/smart-chunk-split.js";
import streamResponse from "./utils/stream-response.js";
const OPEN_MODE = "r";
export default class DownloadEngineFetchStreamLocalFile extends BaseDownloadEngineFetchStream {
transferAction = "Copying";
_fd = null;
_fsPath = null;
withSubState(state) {
const fetchStream = new DownloadEngineFetchStreamLocalFile(this.options);
return this.cloneState(state, fetchStream);
}
async _ensureFileOpen(path) {
return await withLock(this, "_lock", async () => {
if (this._fd && this._fsPath === path) {
return this._fd;
}
this._fd?.close();
return await retry(async () => {
await fsExtra.ensureFile(path);
return this._fd = await fs.open(path, OPEN_MODE);
}, this.options.retry);
});
}
async fetchWithoutRetryChunks(callback) {
const file = await this._ensureFileOpen(this.state.url);
const stream = file.createReadStream({
start: this._startSize,
end: this._endSize - 1,
autoClose: true
});
return await streamResponse(stream, this, new SmartChunkSplit(callback, this.state), this.state.onProgress);
}
async fetchDownloadInfoWithoutRetry(path) {
const stat = await fs.stat(path);
if (!stat.isFile()) {
throw new Error("Path is a directory");
}
return {
length: stat.size,
acceptRange: true
};
}
close() {
super.close();
this._fd?.close();
this._fd = null;
}
}
//# sourceMappingURL=download-engine-fetch-stream-local-file.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"download-engine-fetch-stream-local-file.js","sourceRoot":"","sources":["../../../../../src/download/download-engine/streams/download-engine-fetch-stream/download-engine-fetch-stream-local-file.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAC,QAAQ,EAAC,MAAM,iBAAiB,CAAC;AACzC,OAAO,KAAK,MAAM,aAAa,CAAC;AAChC,OAAO,OAAO,MAAM,UAAU,CAAC;AAC/B,OAAO,6BAAmF,MAAM,wCAAwC,CAAC;AACzI,OAAO,eAAe,MAAM,8BAA8B,CAAC;AAC3D,OAAO,cAAc,MAAM,4BAA4B,CAAC;AAExD,MAAM,SAAS,GAAG,GAAG,CAAC;AAEtB,MAAM,CAAC,OAAO,OAAO,kCAAmC,SAAQ,6BAA6B;IACzE,cAAc,GAAG,SAAS,CAAC;IACnC,GAAG,GAAsB,IAAI,CAAC;IAC9B,OAAO,GAAkB,IAAI,CAAC;IAE7B,YAAY,CAAC,KAAoB;QACtC,MAAM,WAAW,GAAG,IAAI,kCAAkC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACzE,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,WAAW,CAAS,CAAC;IACvD,CAAC;IAEO,KAAK,CAAC,eAAe,CAAC,IAAY;QACtC,OAAO,MAAM,QAAQ,CAAC,IAAI,EAAE,OAAO,EAAE,KAAK,IAAI,EAAE;YAC5C,IAAI,IAAI,CAAC,GAAG,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,EAAE,CAAC;gBACpC,OAAO,IAAI,CAAC,GAAG,CAAC;YACpB,CAAC;YAED,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC;YAClB,OAAO,MAAM,KAAK,CAAC,KAAK,IAAI,EAAE;gBAC1B,MAAM,OAAO,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC/B,OAAO,IAAI,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YACrD,CAAC,EAAE,IAAI,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;QAC3B,CAAC,CAAC,CAAC;IACP,CAAC;IAEkB,KAAK,CAAC,uBAAuB,CAAC,QAAuB;QACpE,MAAM,IAAI,GAAG,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAExD,MAAM,MAAM,GAAG,IAAI,CAAC,gBAAgB,CAAC;YACjC,KAAK,EAAE,IAAI,CAAC,UAAU;YACtB,GAAG,EAAE,IAAI,CAAC,QAAQ,GAAG,CAAC;YACtB,SAAS,EAAE,IAAI;SAClB,CAAC,CAAC;QAEH,OAAO,MAAM,cAAc,CAAC,MAAM,EAAE,IAAI,EAAE,IAAI,eAAe,CAAC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IAChH,CAAC;IAEkB,KAAK,CAAC,6BAA6B,CAAC,IAAY;QAC/D,MAAM,IAAI,GAAG,MAAM,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;YACjB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;QAC3C,CAAC;QACD,OAAO;YACH,MAAM,EAAE,IAAI,CAAC,IAAI;YACjB,WAAW,EAAE,IAAI;SACpB,CAAC;IACN,CAAC;IAEQ,KAAK;QACV,KAAK,CAAC,KAAK,EAAE,CAAC;QACd,IAAI,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC;QAClB,IAAI,CAAC,GAAG,GAAG,IAAI,CAAC;IACpB,CAAC;CACJ"}

View File

@@ -0,0 +1,18 @@
import BaseDownloadEngineFetchStream, { DownloadInfoResponse, FetchSubState, WriteCallback } from "./base-download-engine-fetch-stream.js";
import { AvailablePrograms } from "../../download-file/download-programs/switch-program.js";
export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetchStream {
private _fetchDownloadInfoWithHEAD;
readonly programType: AvailablePrograms;
transferAction: string;
withSubState(state: FetchSubState): this;
fetchBytes(url: string, start: number, end: number, onProgress?: (length: number) => void): Promise<Uint8Array>;
protected fetchBytesWithoutRetry(url: string, start: number, end: number, onProgress?: (length: number) => void): Promise<Uint8Array>;
fetchChunks(callback: WriteCallback): Promise<void>;
protected fetchWithoutRetryChunks(): Promise<void>;
protected _fetchChunksRangeSupport(callback: WriteCallback): Promise<void>;
protected _fetchChunksWithoutRange(callback: WriteCallback): Promise<void>;
protected fetchDownloadInfoWithoutRetry(url: string): Promise<DownloadInfoResponse>;
protected fetchDownloadInfoWithoutRetryByMethod(url: string, method?: "HEAD" | "GET"): Promise<DownloadInfoResponse>;
protected fetchDownloadInfoWithoutRetryContentRange(url: string, xhrResponse?: XMLHttpRequest): number | Promise<number>;
protected static convertXHRHeadersToRecord(xhr: XMLHttpRequest): Record<string, string>;
}

View File

@@ -0,0 +1,196 @@
import BaseDownloadEngineFetchStream, { MIN_LENGTH_FOR_MORE_INFO_REQUEST } from "./base-download-engine-fetch-stream.js";
import EmptyResponseError from "./errors/empty-response-error.js";
import StatusCodeError from "./errors/status-code-error.js";
import XhrError from "./errors/xhr-error.js";
import InvalidContentLengthError from "./errors/invalid-content-length-error.js";
import retry from "async-retry";
import { parseContentDisposition } from "./utils/content-disposition.js";
import { parseHttpContentRange } from "./utils/httpRange.js";
export default class DownloadEngineFetchStreamXhr extends BaseDownloadEngineFetchStream {
_fetchDownloadInfoWithHEAD = true;
programType = "chunks";
transferAction = "Downloading";
withSubState(state) {
const fetchStream = new DownloadEngineFetchStreamXhr(this.options);
return this.cloneState(state, fetchStream);
}
async fetchBytes(url, start, end, onProgress) {
return await retry(async () => {
return await this.fetchBytesWithoutRetry(url, start, end, onProgress);
}, this.options.retry);
}
fetchBytesWithoutRetry(url, start, end, onProgress) {
return new Promise((resolve, reject) => {
const headers = {
accept: "*/*",
...this.options.headers
};
if (this.state.rangeSupport) {
headers.range = `bytes=${start}-${end - 1}`;
}
const xhr = new XMLHttpRequest();
xhr.responseType = "arraybuffer";
xhr.open("GET", this.appendToURL(url), true);
for (const [key, value] of Object.entries(headers)) {
xhr.setRequestHeader(key, value);
}
xhr.onload = () => {
const contentLength = parseInt(xhr.getResponseHeader("content-length"));
if (this.state.rangeSupport && contentLength !== end - start) {
throw new InvalidContentLengthError(end - start, contentLength);
}
if (xhr.status >= 200 && xhr.status < 300) {
const arrayBuffer = xhr.response;
if (arrayBuffer) {
resolve(new Uint8Array(arrayBuffer));
}
else {
reject(new EmptyResponseError(url, headers));
}
}
else {
reject(new StatusCodeError(url, xhr.status, xhr.statusText, headers));
}
};
xhr.onerror = () => {
reject(new XhrError(`Failed to fetch ${url}`));
};
xhr.onprogress = (event) => {
if (event.lengthComputable) {
onProgress?.(event.loaded);
}
};
xhr.send();
this.on("aborted", () => {
xhr.abort();
});
});
}
async fetchChunks(callback) {
if (this.state.rangeSupport) {
return await this._fetchChunksRangeSupport(callback);
}
return await this._fetchChunksWithoutRange(callback);
}
fetchWithoutRetryChunks() {
throw new Error("Method not needed, use fetchChunks instead.");
}
async _fetchChunksRangeSupport(callback) {
while (this._startSize < this._endSize) {
await this.paused;
if (this.aborted)
return;
const chunk = await this.fetchBytes(this.state.url, this._startSize, this._endSize, this.state.onProgress);
callback([chunk], this._startSize, this.state.startChunk++);
}
}
async _fetchChunksWithoutRange(callback) {
const relevantContent = await (async () => {
const result = await this.fetchBytes(this.state.url, 0, this._endSize, this.state.onProgress);
return result.slice(this._startSize, this._endSize || result.length);
})();
let totalReceivedLength = 0;
let index = 0;
while (totalReceivedLength < relevantContent.byteLength) {
await this.paused;
if (this.aborted)
return;
const start = totalReceivedLength;
const end = Math.min(relevantContent.byteLength, start + this.state.chunkSize);
const chunk = relevantContent.slice(start, end);
totalReceivedLength += chunk.byteLength;
callback([chunk], index * this.state.chunkSize, index++);
}
}
fetchDownloadInfoWithoutRetry(url) {
if (this._fetchDownloadInfoWithHEAD) {
try {
return this.fetchDownloadInfoWithoutRetryByMethod(url, "HEAD");
}
catch (error) {
if (!(error instanceof StatusCodeError)) {
throw error;
}
this._fetchDownloadInfoWithHEAD = false;
}
}
return this.fetchDownloadInfoWithoutRetryByMethod(url, "GET");
}
async fetchDownloadInfoWithoutRetryByMethod(url, method = "HEAD") {
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open(method, url, true);
const allHeaders = {
...this.options.headers
};
for (const [key, value] of Object.entries(allHeaders)) {
xhr.setRequestHeader(key, value);
}
xhr.onload = async () => {
if (xhr.status >= 200 && xhr.status < 300) {
const contentLength = parseInt(xhr.getResponseHeader("content-length"));
const length = MIN_LENGTH_FOR_MORE_INFO_REQUEST < contentLength ? await this.fetchDownloadInfoWithoutRetryContentRange(url, method === "GET" ? xhr : undefined) : 0;
const fileName = parseContentDisposition(xhr.getResponseHeader("content-disposition"));
const acceptRange = this.options.acceptRangeIsKnown ?? xhr.getResponseHeader("Accept-Ranges") === "bytes";
resolve({
length,
acceptRange,
newURL: xhr.responseURL,
fileName
});
}
else {
reject(new StatusCodeError(url, xhr.status, xhr.statusText, this.options.headers, DownloadEngineFetchStreamXhr.convertXHRHeadersToRecord(xhr)));
}
};
xhr.onerror = function () {
reject(new XhrError(`Failed to fetch ${url}`));
};
xhr.send();
});
}
fetchDownloadInfoWithoutRetryContentRange(url, xhrResponse) {
const getSize = (xhr) => {
const contentRange = xhr.getResponseHeader("Content-Range");
return parseHttpContentRange(contentRange)?.size || 0;
};
if (xhrResponse) {
return getSize(xhrResponse);
}
return new Promise((resolve, reject) => {
const xhr = new XMLHttpRequest();
xhr.open("GET", url, true);
const allHeaders = {
accept: "*/*",
...this.options.headers,
range: "bytes=0-0"
};
for (const [key, value] of Object.entries(allHeaders)) {
xhr.setRequestHeader(key, value);
}
xhr.onload = () => {
resolve(getSize(xhr));
};
xhr.onerror = () => {
reject(new XhrError(`Failed to fetch ${url}`));
};
xhr.send();
});
}
static convertXHRHeadersToRecord(xhr) {
const headersString = xhr.getAllResponseHeaders();
const headersArray = headersString.trim()
.split(/[\r\n]+/);
const headersObject = {};
headersArray.forEach(line => {
const parts = line.split(": ");
const key = parts.shift();
const value = parts.join(": ");
if (key) {
headersObject[key] = value;
}
});
return headersObject;
}
}
//# sourceMappingURL=download-engine-fetch-stream-xhr.js.map

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,9 @@
import FetchStreamError from "./fetch-stream-error.js";
export default class EmptyResponseError extends FetchStreamError {
readonly headers: {
[key: string]: string | string[];
};
constructor(url: string, headers: {
[key: string]: string | string[];
});
}

View File

@@ -0,0 +1,9 @@
import FetchStreamError from "./fetch-stream-error.js";
export default class EmptyResponseError extends FetchStreamError {
headers;
constructor(url, headers) {
super("Empty response error: " + url);
this.headers = headers;
}
}
//# sourceMappingURL=empty-response-error.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"empty-response-error.js","sourceRoot":"","sources":["../../../../../../src/download/download-engine/streams/download-engine-fetch-stream/errors/empty-response-error.ts"],"names":[],"mappings":"AAAA,OAAO,gBAAgB,MAAM,yBAAyB,CAAC;AAEvD,MAAM,CAAC,OAAO,OAAO,kBAAmB,SAAQ,gBAAgB;IACnB;IAAzC,YAAY,GAAW,EAAkB,OAA6C;QAClF,KAAK,CAAC,wBAAwB,GAAG,GAAG,CAAC,CAAC;QADD,YAAO,GAAP,OAAO,CAAsC;IAEtF,CAAC;CACJ"}

View File

@@ -0,0 +1,3 @@
import IpullError from "../../../../../errors/ipull-error.js";
export default class FetchStreamError extends IpullError {
}

View File

@@ -0,0 +1,4 @@
import IpullError from "../../../../../errors/ipull-error.js";
export default class FetchStreamError extends IpullError {
}
//# sourceMappingURL=fetch-stream-error.js.map

View File

@@ -0,0 +1 @@
{"version":3,"file":"fetch-stream-error.js","sourceRoot":"","sources":["../../../../../../src/download/download-engine/streams/download-engine-fetch-stream/errors/fetch-stream-error.ts"],"names":[],"mappings":"AAAA,OAAO,UAAU,MAAM,sCAAsC,CAAC;AAE9D,MAAM,CAAC,OAAO,OAAO,gBAAiB,SAAQ,UAAU;CAAG"}

View File

@@ -0,0 +1,6 @@
import FetchStreamError from "./fetch-stream-error.js";
/**
* Represents an error that return from the server (will not retry)
*/
export default class HttpError extends FetchStreamError {
}

View File

@@ -0,0 +1,7 @@
import FetchStreamError from "./fetch-stream-error.js";
/**
* Represents an error that return from the server (will not retry)
*/
export default class HttpError extends FetchStreamError {
}
//# sourceMappingURL=http-error.js.map

Some files were not shown because too many files have changed in this diff Show More