Upakovka v Electron.JS no po staroy sborke cherez .cjs

This commit is contained in:
Neyra
2026-02-10 19:19:41 +08:00
parent 8e9b7201ed
commit a1bba1d3d1
442 changed files with 19825 additions and 47462 deletions

View File

@@ -49,6 +49,48 @@ function DBCSCodec(codecOptions, iconv) {
for (var i = 0; i < mappingTable.length; i++)
this._addDecodeChunk(mappingTable[i]);
// Load & create GB18030 tables when needed.
if (typeof codecOptions.gb18030 === 'function') {
this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.
// Add GB18030 common decode nodes.
var commonThirdByteNodeIdx = this.decodeTables.length;
this.decodeTables.push(UNASSIGNED_NODE.slice(0));
var commonFourthByteNodeIdx = this.decodeTables.length;
this.decodeTables.push(UNASSIGNED_NODE.slice(0));
// Fill out the tree
var firstByteNode = this.decodeTables[0];
for (var i = 0x81; i <= 0xFE; i++) {
var secondByteNode = this.decodeTables[NODE_START - firstByteNode[i]];
for (var j = 0x30; j <= 0x39; j++) {
if (secondByteNode[j] === UNASSIGNED) {
secondByteNode[j] = NODE_START - commonThirdByteNodeIdx;
} else if (secondByteNode[j] > NODE_START) {
throw new Error("gb18030 decode tables conflict at byte 2");
}
var thirdByteNode = this.decodeTables[NODE_START - secondByteNode[j]];
for (var k = 0x81; k <= 0xFE; k++) {
if (thirdByteNode[k] === UNASSIGNED) {
thirdByteNode[k] = NODE_START - commonFourthByteNodeIdx;
} else if (thirdByteNode[k] === NODE_START - commonFourthByteNodeIdx) {
continue;
} else if (thirdByteNode[k] > NODE_START) {
throw new Error("gb18030 decode tables conflict at byte 3");
}
var fourthByteNode = this.decodeTables[NODE_START - thirdByteNode[k]];
for (var l = 0x30; l <= 0x39; l++) {
if (fourthByteNode[l] === UNASSIGNED)
fourthByteNode[l] = GB18030_CODE;
}
}
}
}
}
this.defaultCharUnicode = iconv.defaultCharUnicode;
@@ -92,30 +134,6 @@ function DBCSCodec(codecOptions, iconv) {
this.defCharSB = this.encodeTable[0][iconv.defaultCharSingleByte.charCodeAt(0)];
if (this.defCharSB === UNASSIGNED) this.defCharSB = this.encodeTable[0]['?'];
if (this.defCharSB === UNASSIGNED) this.defCharSB = "?".charCodeAt(0);
// Load & create GB18030 tables when needed.
if (typeof codecOptions.gb18030 === 'function') {
this.gb18030 = codecOptions.gb18030(); // Load GB18030 ranges.
// Add GB18030 decode tables.
var thirdByteNodeIdx = this.decodeTables.length;
var thirdByteNode = this.decodeTables[thirdByteNodeIdx] = UNASSIGNED_NODE.slice(0);
var fourthByteNodeIdx = this.decodeTables.length;
var fourthByteNode = this.decodeTables[fourthByteNodeIdx] = UNASSIGNED_NODE.slice(0);
for (var i = 0x81; i <= 0xFE; i++) {
var secondByteNodeIdx = NODE_START - this.decodeTables[0][i];
var secondByteNode = this.decodeTables[secondByteNodeIdx];
for (var j = 0x30; j <= 0x39; j++)
secondByteNode[j] = NODE_START - thirdByteNodeIdx;
}
for (var i = 0x81; i <= 0xFE; i++)
thirdByteNode[i] = NODE_START - fourthByteNodeIdx;
for (var i = 0x30; i <= 0x39; i++)
fourthByteNode[i] = GB18030_CODE
}
}
DBCSCodec.prototype.encoder = DBCSEncoder;
@@ -124,7 +142,7 @@ DBCSCodec.prototype.decoder = DBCSDecoder;
// Decoder helpers
DBCSCodec.prototype._getDecodeTrieNode = function(addr) {
var bytes = [];
for (; addr > 0; addr >>= 8)
for (; addr > 0; addr >>>= 8)
bytes.push(addr & 0xFF);
if (bytes.length == 0)
bytes.push(0);
@@ -249,19 +267,32 @@ DBCSCodec.prototype._setEncodeSequence = function(seq, dbcsCode) {
DBCSCodec.prototype._fillEncodeTable = function(nodeIdx, prefix, skipEncodeChars) {
var node = this.decodeTables[nodeIdx];
var hasValues = false;
var subNodeEmpty = {};
for (var i = 0; i < 0x100; i++) {
var uCode = node[i];
var mbCode = prefix + i;
if (skipEncodeChars[mbCode])
continue;
if (uCode >= 0)
if (uCode >= 0) {
this._setEncodeChar(uCode, mbCode);
else if (uCode <= NODE_START)
this._fillEncodeTable(NODE_START - uCode, mbCode << 8, skipEncodeChars);
else if (uCode <= SEQ_START)
hasValues = true;
} else if (uCode <= NODE_START) {
var subNodeIdx = NODE_START - uCode;
if (!subNodeEmpty[subNodeIdx]) { // Skip empty subtrees (they are too large in gb18030).
var newPrefix = (mbCode << 8) >>> 0; // NOTE: '>>> 0' keeps 32-bit num positive.
if (this._fillEncodeTable(subNodeIdx, newPrefix, skipEncodeChars))
hasValues = true;
else
subNodeEmpty[subNodeIdx] = true;
}
} else if (uCode <= SEQ_START) {
this._setEncodeSequence(this.decodeTableSeq[SEQ_START - uCode], mbCode);
hasValues = true;
}
}
return hasValues;
}
@@ -388,10 +419,15 @@ DBCSEncoder.prototype.write = function(str) {
newBuf[j++] = dbcsCode >> 8; // high byte
newBuf[j++] = dbcsCode & 0xFF; // low byte
}
else {
else if (dbcsCode < 0x1000000) {
newBuf[j++] = dbcsCode >> 16;
newBuf[j++] = (dbcsCode >> 8) & 0xFF;
newBuf[j++] = dbcsCode & 0xFF;
} else {
newBuf[j++] = dbcsCode >>> 24;
newBuf[j++] = (dbcsCode >>> 16) & 0xFF;
newBuf[j++] = (dbcsCode >>> 8) & 0xFF;
newBuf[j++] = dbcsCode & 0xFF;
}
}
@@ -440,7 +476,7 @@ DBCSEncoder.prototype.findIdx = findIdx;
function DBCSDecoder(options, codec) {
// Decoder state
this.nodeIdx = 0;
this.prevBuf = Buffer.alloc(0);
this.prevBytes = [];
// Static data
this.decodeTables = codec.decodeTables;
@@ -452,15 +488,12 @@ function DBCSDecoder(options, codec) {
DBCSDecoder.prototype.write = function(buf) {
var newBuf = Buffer.alloc(buf.length*2),
nodeIdx = this.nodeIdx,
prevBuf = this.prevBuf, prevBufOffset = this.prevBuf.length,
seqStart = -this.prevBuf.length, // idx of the start of current parsed sequence.
prevBytes = this.prevBytes, prevOffset = this.prevBytes.length,
seqStart = -this.prevBytes.length, // idx of the start of current parsed sequence.
uCode;
if (prevBufOffset > 0) // Make prev buf overlap a little to make it easier to slice later.
prevBuf = Buffer.concat([prevBuf, buf.slice(0, 10)]);
for (var i = 0, j = 0; i < buf.length; i++) {
var curByte = (i >= 0) ? buf[i] : prevBuf[i + prevBufOffset];
var curByte = (i >= 0) ? buf[i] : prevBytes[i + prevOffset];
// Lookup in current trie node.
var uCode = this.decodeTables[nodeIdx][curByte];
@@ -470,13 +503,18 @@ DBCSDecoder.prototype.write = function(buf) {
}
else if (uCode === UNASSIGNED) { // Unknown char.
// TODO: Callback with seq.
//var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset);
i = seqStart; // Try to parse again, after skipping first byte of the sequence ('i' will be incremented by 'for' cycle).
uCode = this.defaultCharUnicode.charCodeAt(0);
i = seqStart; // Skip one byte ('i' will be incremented by the for loop) and try to parse again.
}
else if (uCode === GB18030_CODE) {
var curSeq = (seqStart >= 0) ? buf.slice(seqStart, i+1) : prevBuf.slice(seqStart + prevBufOffset, i+1 + prevBufOffset);
var ptr = (curSeq[0]-0x81)*12600 + (curSeq[1]-0x30)*1260 + (curSeq[2]-0x81)*10 + (curSeq[3]-0x30);
if (i >= 3) {
var ptr = (buf[i-3]-0x81)*12600 + (buf[i-2]-0x30)*1260 + (buf[i-1]-0x81)*10 + (curByte-0x30);
} else {
var ptr = (prevBytes[i-3+prevOffset]-0x81)*12600 +
(((i-2 >= 0) ? buf[i-2] : prevBytes[i-2+prevOffset])-0x30)*1260 +
(((i-1 >= 0) ? buf[i-1] : prevBytes[i-1+prevOffset])-0x81)*10 +
(curByte-0x30);
}
var idx = findIdx(this.gb18030.gbChars, ptr);
uCode = this.gb18030.uChars[idx] + ptr - this.gb18030.gbChars[idx];
}
@@ -497,13 +535,13 @@ DBCSDecoder.prototype.write = function(buf) {
throw new Error("iconv-lite internal error: invalid decoding table value " + uCode + " at " + nodeIdx + "/" + curByte);
// Write the character to buffer, handling higher planes using surrogate pair.
if (uCode > 0xFFFF) {
if (uCode >= 0x10000) {
uCode -= 0x10000;
var uCodeLead = 0xD800 + Math.floor(uCode / 0x400);
var uCodeLead = 0xD800 | (uCode >> 10);
newBuf[j++] = uCodeLead & 0xFF;
newBuf[j++] = uCodeLead >> 8;
uCode = 0xDC00 + uCode % 0x400;
uCode = 0xDC00 | (uCode & 0x3FF);
}
newBuf[j++] = uCode & 0xFF;
newBuf[j++] = uCode >> 8;
@@ -513,7 +551,10 @@ DBCSDecoder.prototype.write = function(buf) {
}
this.nodeIdx = nodeIdx;
this.prevBuf = (seqStart >= 0) ? buf.slice(seqStart) : prevBuf.slice(seqStart + prevBufOffset);
this.prevBytes = (seqStart >= 0)
? Array.prototype.slice.call(buf, seqStart)
: prevBytes.slice(seqStart + prevOffset).concat(Array.prototype.slice.call(buf));
return newBuf.slice(0, j).toString('ucs2');
}
@@ -521,18 +562,19 @@ DBCSDecoder.prototype.end = function() {
var ret = '';
// Try to parse all remaining chars.
while (this.prevBuf.length > 0) {
while (this.prevBytes.length > 0) {
// Skip 1 character in the buffer.
ret += this.defaultCharUnicode;
var buf = this.prevBuf.slice(1);
var bytesArr = this.prevBytes.slice(1);
// Parse remaining as usual.
this.prevBuf = Buffer.alloc(0);
this.prevBytes = [];
this.nodeIdx = 0;
if (buf.length > 0)
ret += this.write(buf);
if (bytesArr.length > 0)
ret += this.write(bytesArr);
}
this.prevBytes = [];
this.nodeIdx = 0;
return ret;
}
@@ -544,7 +586,7 @@ function findIdx(table, val) {
var l = 0, r = table.length;
while (l < r-1) { // always table[l] <= val < table[r]
var mid = l + Math.floor((r-l+1)/2);
var mid = l + ((r-l+1) >> 1);
if (table[mid] <= val)
l = mid;
else

View File

@@ -167,7 +167,19 @@ module.exports = {
'big5hkscs': {
type: '_dbcs',
table: function() { return require('./tables/cp950.json').concat(require('./tables/big5-added.json')) },
encodeSkipVals: [0xa2cc],
encodeSkipVals: [
// Although Encoding Standard says we should avoid encoding to HKSCS area (See Step 1 of
// https://encoding.spec.whatwg.org/#index-big5-pointer), we still do it to increase compatibility with ICU.
// But if a single unicode point can be encoded both as HKSCS and regular Big5, we prefer the latter.
0x8e69, 0x8e6f, 0x8e7e, 0x8eab, 0x8eb4, 0x8ecd, 0x8ed0, 0x8f57, 0x8f69, 0x8f6e, 0x8fcb, 0x8ffe,
0x906d, 0x907a, 0x90c4, 0x90dc, 0x90f1, 0x91bf, 0x92af, 0x92b0, 0x92b1, 0x92b2, 0x92d1, 0x9447, 0x94ca,
0x95d9, 0x96fc, 0x9975, 0x9b76, 0x9b78, 0x9b7b, 0x9bc6, 0x9bde, 0x9bec, 0x9bf6, 0x9c42, 0x9c53, 0x9c62,
0x9c68, 0x9c6b, 0x9c77, 0x9cbc, 0x9cbd, 0x9cd0, 0x9d57, 0x9d5a, 0x9dc4, 0x9def, 0x9dfb, 0x9ea9, 0x9eef,
0x9efd, 0x9f60, 0x9fcb, 0xa077, 0xa0dc, 0xa0df, 0x8fcc, 0x92c8, 0x9644, 0x96ed,
// Step 2 of https://encoding.spec.whatwg.org/#index-big5-pointer: Use last pointer for U+2550, U+255E, U+2561, U+256A, U+5341, or U+5345
0xa2a4, 0xa2a5, 0xa2a7, 0xa2a6, 0xa2cc, 0xa2ce,
],
},
'cnbig5': 'big5hkscs',

View File

@@ -4,6 +4,7 @@
// We support Browserify by skipping automatic module discovery and requiring modules directly.
var modules = [
require("./internal"),
require("./utf32"),
require("./utf16"),
require("./utf7"),
require("./sbcs-codec"),
@@ -13,7 +14,7 @@ var modules = [
require("./dbcs-data"),
];
// Put all encoding/alias/codec definitions to single object and export it.
// Put all encoding/alias/codec definitions to single object and export it.
for (var i = 0; i < modules.length; i++) {
var module = modules[i];
for (var enc in module)

View File

@@ -53,10 +53,20 @@ if (!StringDecoder.prototype.end) // Node v0.8 doesn't have this method.
function InternalDecoder(options, codec) {
StringDecoder.call(this, codec.enc);
this.decoder = new StringDecoder(codec.enc);
}
InternalDecoder.prototype = StringDecoder.prototype;
InternalDecoder.prototype.write = function(buf) {
if (!Buffer.isBuffer(buf)) {
buf = Buffer.from(buf);
}
return this.decoder.write(buf);
}
InternalDecoder.prototype.end = function() {
return this.decoder.end();
}
//------------------------------------------------------------------------------

View File

@@ -22,6 +22,11 @@ module.exports = {
"chars": "АБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯабвгдежзийклмнопрстуфхцчшщъыьэюя└┴┬├─┼╣║╚╔╩╦╠═╬┐░▒▓│┤№§╗╝┘┌█▄▌▐▀αßΓπΣσµτΦΘΩδ∞φε∩≡±≥≤⌠⌡÷≈°∙·√ⁿ²■ "
},
"cp720": {
"type": "_sbcs",
"chars": "\x80\x81éâ\x84à\x86çêëèïî\x8d\x8e\x8f\x90\u0651\u0652ô¤ـûùءآأؤ£إئابةتثجحخدذرزسشص«»░▒▓│┤╡╢╖╕╣║╗╝╜╛┐└┴┬├─┼╞╟╚╔╩╦╠═╬╧╨╤╥╙╘╒╓╫╪┘┌█▄▌▐▀ضطظعغفµقكلمنهوىي≡\u064b\u064c\u064d\u064e\u064f\u0650≈°∙·√ⁿ²■\u00a0"
},
// Aliases of generated encodings.
"ascii8bit": "ascii",
"usascii": "ascii",

View File

@@ -27,7 +27,7 @@
["a7c2","",14],
["a7f2","",12],
["a896","",10],
["a8bc",""],
["a8bc","ḿ"],
["a8bf","ǹ"],
["a8c1",""],
["a8ea","",20],
@@ -51,5 +51,6 @@
["fca1","",93],
["fda1","",93],
["fe50","⺁⺄㑳㑇⺈⺋㖞㘚㘎⺌⺗㥮㤘㧏㧟㩳㧐㭎㱮㳠⺧⺪䁖䅟⺮䌷⺳⺶⺷䎱䎬⺻䏝䓖䙡䙌"],
["fe80","䜣䜩䝼䞍⻊䥇䥺䥽䦂䦃䦅䦆䦟䦛䦷䦶䲣䲟䲠䲡䱷䲢䴓",6,"䶮",93]
["fe80","䜣䜩䝼䞍⻊䥇䥺䥽䦂䦃䦅䦆䦟䦛䦷䦶䲣䲟䲠䲡䱷䲢䴓",6,"䶮",93],
["8135f437",""]
]

View File

@@ -61,6 +61,7 @@ Utf16BEDecoder.prototype.write = function(buf) {
}
Utf16BEDecoder.prototype.end = function() {
this.overflowByte = -1;
}
@@ -103,8 +104,8 @@ Utf16Encoder.prototype.end = function() {
function Utf16Decoder(options, codec) {
this.decoder = null;
this.initialBytes = [];
this.initialBytesLen = 0;
this.initialBufs = [];
this.initialBufsLen = 0;
this.options = options || {};
this.iconv = codec.iconv;
@@ -113,17 +114,22 @@ function Utf16Decoder(options, codec) {
Utf16Decoder.prototype.write = function(buf) {
if (!this.decoder) {
// Codec is not chosen yet. Accumulate initial bytes.
this.initialBytes.push(buf);
this.initialBytesLen += buf.length;
this.initialBufs.push(buf);
this.initialBufsLen += buf.length;
if (this.initialBytesLen < 16) // We need more bytes to use space heuristic (see below)
if (this.initialBufsLen < 16) // We need more bytes to use space heuristic (see below)
return '';
// We have enough bytes -> detect endianness.
var buf = Buffer.concat(this.initialBytes),
encoding = detectEncoding(buf, this.options.defaultEncoding);
var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding);
this.decoder = this.iconv.getDecoder(encoding, this.options);
this.initialBytes.length = this.initialBytesLen = 0;
var resStr = '';
for (var i = 0; i < this.initialBufs.length; i++)
resStr += this.decoder.write(this.initialBufs[i]);
this.initialBufs.length = this.initialBufsLen = 0;
return resStr;
}
return this.decoder.write(buf);
@@ -131,47 +137,61 @@ Utf16Decoder.prototype.write = function(buf) {
Utf16Decoder.prototype.end = function() {
if (!this.decoder) {
var buf = Buffer.concat(this.initialBytes),
encoding = detectEncoding(buf, this.options.defaultEncoding);
var encoding = detectEncoding(this.initialBufs, this.options.defaultEncoding);
this.decoder = this.iconv.getDecoder(encoding, this.options);
var res = this.decoder.write(buf),
trail = this.decoder.end();
var resStr = '';
for (var i = 0; i < this.initialBufs.length; i++)
resStr += this.decoder.write(this.initialBufs[i]);
return trail ? (res + trail) : res;
var trail = this.decoder.end();
if (trail)
resStr += trail;
this.initialBufs.length = this.initialBufsLen = 0;
return resStr;
}
return this.decoder.end();
}
function detectEncoding(buf, defaultEncoding) {
var enc = defaultEncoding || 'utf-16le';
function detectEncoding(bufs, defaultEncoding) {
var b = [];
var charsProcessed = 0;
var asciiCharsLE = 0, asciiCharsBE = 0; // Number of ASCII chars when decoded as LE or BE.
if (buf.length >= 2) {
// Check BOM.
if (buf[0] == 0xFE && buf[1] == 0xFF) // UTF-16BE BOM
enc = 'utf-16be';
else if (buf[0] == 0xFF && buf[1] == 0xFE) // UTF-16LE BOM
enc = 'utf-16le';
else {
// No BOM found. Try to deduce encoding from initial content.
// Most of the time, the content has ASCII chars (U+00**), but the opposite (U+**00) is uncommon.
// So, we count ASCII as if it was LE or BE, and decide from that.
var asciiCharsLE = 0, asciiCharsBE = 0, // Counts of chars in both positions
_len = Math.min(buf.length - (buf.length % 2), 64); // Len is always even.
outer_loop:
for (var i = 0; i < bufs.length; i++) {
var buf = bufs[i];
for (var j = 0; j < buf.length; j++) {
b.push(buf[j]);
if (b.length === 2) {
if (charsProcessed === 0) {
// Check BOM first.
if (b[0] === 0xFF && b[1] === 0xFE) return 'utf-16le';
if (b[0] === 0xFE && b[1] === 0xFF) return 'utf-16be';
}
for (var i = 0; i < _len; i += 2) {
if (buf[i] === 0 && buf[i+1] !== 0) asciiCharsBE++;
if (buf[i] !== 0 && buf[i+1] === 0) asciiCharsLE++;
if (b[0] === 0 && b[1] !== 0) asciiCharsBE++;
if (b[0] !== 0 && b[1] === 0) asciiCharsLE++;
b.length = 0;
charsProcessed++;
if (charsProcessed >= 100) {
break outer_loop;
}
}
if (asciiCharsBE > asciiCharsLE)
enc = 'utf-16be';
else if (asciiCharsBE < asciiCharsLE)
enc = 'utf-16le';
}
}
return enc;
// Make decisions.
// Most of the time, the content has ASCII chars (U+00**), but the opposite (U+**00) is uncommon.
// So, we count ASCII as if it was LE or BE, and decide from that.
if (asciiCharsBE > asciiCharsLE) return 'utf-16be';
if (asciiCharsBE < asciiCharsLE) return 'utf-16le';
// Couldn't decide (likely all zeros or not enough data).
return defaultEncoding || 'utf-16le';
}

View File

@@ -74,7 +74,7 @@ Utf7Decoder.prototype.write = function(buf) {
if (i == lastI && buf[i] == minusChar) {// "+-" -> "+"
res += "+";
} else {
var b64str = base64Accum + buf.slice(lastI, i).toString();
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI, i), "ascii");
res += this.iconv.decode(Buffer.from(b64str, 'base64'), "utf16-be");
}
@@ -91,7 +91,7 @@ Utf7Decoder.prototype.write = function(buf) {
if (!inBase64) {
res += this.iconv.decode(buf.slice(lastI), "ascii"); // Write direct chars.
} else {
var b64str = base64Accum + buf.slice(lastI).toString();
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI), "ascii");
var canBeDecoded = b64str.length - (b64str.length % 8); // Minimal chunk: 2 quads -> 2x3 bytes -> 3 chars.
base64Accum = b64str.slice(canBeDecoded); // The rest will be decoded in future.
@@ -245,7 +245,7 @@ Utf7IMAPDecoder.prototype.write = function(buf) {
if (i == lastI && buf[i] == minusChar) { // "&-" -> "&"
res += "&";
} else {
var b64str = base64Accum + buf.slice(lastI, i).toString().replace(/,/g, '/');
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI, i), "ascii").replace(/,/g, '/');
res += this.iconv.decode(Buffer.from(b64str, 'base64'), "utf16-be");
}
@@ -262,7 +262,7 @@ Utf7IMAPDecoder.prototype.write = function(buf) {
if (!inBase64) {
res += this.iconv.decode(buf.slice(lastI), "ascii"); // Write direct chars.
} else {
var b64str = base64Accum + buf.slice(lastI).toString().replace(/,/g, '/');
var b64str = base64Accum + this.iconv.decode(buf.slice(lastI), "ascii").replace(/,/g, '/');
var canBeDecoded = b64str.length - (b64str.length % 8); // Minimal chunk: 2 quads -> 2x3 bytes -> 3 chars.
base64Accum = b64str.slice(canBeDecoded); // The rest will be decoded in future.