alyaevgi/airllm-fork-nodejs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

First upload version 0.0.1

Browse Source
This commit is contained in:
Neyra
2026-02-05 15:27:49 +08:00
commit 8e9b7201ed
4182 changed files with 593136 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
node_modules/node-llama-cpp/dist/utils/tokenizerUtils.d.ts generated vendored Normal file
View File

@@ -0,0 +1,12 @@
import { LlamaVocabularyType } from "../bindings/types.js";
import type { LlamaModelTokens } from "../evaluator/LlamaModel/LlamaModel.js";
/**
* Resolve whether a token has to be prepended at the beginning of the input, and what should it be,
* based on the tokenizer implementation in `llama.cpp` under the `llama_tokenize_internal` function in `llama-vocab.cpp`
*/
export declare function resolveBeginningTokenToPrepend(vocabularyType: LlamaVocabularyType, tokens: LlamaModelTokens): import("../types.js").Token | null;
/**
* Resolve whether a token has to be appended at the end of the input, and what should it be,
* based on the tokenizer implementation in `llama.cpp` under the `llama_tokenize_internal` function in `llama-vocab.cpp`
*/
export declare function resolveEndTokenToAppend(vocabularyType: LlamaVocabularyType, tokens: LlamaModelTokens): import("../types.js").Token | null;