141 lines
3.3 KiB
TypeScript
141 lines
3.3 KiB
TypeScript
import { Context, Recogniser } from '.';
|
|
import match, { Match } from '../match';
|
|
|
|
/**
|
|
* This class matches UTF-16 and UTF-32, both big- and little-endian. The
|
|
* BOM will be used if it is present.
|
|
*/
|
|
export class UTF_16BE implements Recogniser {
|
|
name() {
|
|
return 'UTF-16BE';
|
|
}
|
|
|
|
match(det: Context): Match | null {
|
|
const input = det.fRawInput;
|
|
|
|
if (
|
|
input.length >= 2 &&
|
|
(input[0] & 0xff) == 0xfe &&
|
|
(input[1] & 0xff) == 0xff
|
|
) {
|
|
return match(det, this, 100); // confidence = 100
|
|
}
|
|
|
|
// TODO: Do some statistics to check for unsigned UTF-16BE
|
|
return null;
|
|
}
|
|
}
|
|
|
|
export class UTF_16LE implements Recogniser {
|
|
name() {
|
|
return 'UTF-16LE';
|
|
}
|
|
match(det: Context): Match | null {
|
|
const input = det.fRawInput;
|
|
|
|
if (
|
|
input.length >= 2 &&
|
|
(input[0] & 0xff) == 0xff &&
|
|
(input[1] & 0xff) == 0xfe
|
|
) {
|
|
// LE BOM is present.
|
|
if (input.length >= 4 && input[2] == 0x00 && input[3] == 0x00) {
|
|
// It is probably UTF-32 LE, not UTF-16
|
|
return null;
|
|
}
|
|
return match(det, this, 100); // confidence = 100
|
|
}
|
|
|
|
// TODO: Do some statistics to check for unsigned UTF-16LE
|
|
return null;
|
|
}
|
|
}
|
|
|
|
interface WithGetChar {
|
|
getChar(input: Uint8Array, index: number): number;
|
|
}
|
|
|
|
class UTF_32 implements Recogniser, WithGetChar {
|
|
name() {
|
|
return 'UTF-32';
|
|
}
|
|
|
|
getChar(input: Uint8Array, index: number): number {
|
|
return -1;
|
|
}
|
|
|
|
match(det: Context): Match | null {
|
|
let numValid = 0,
|
|
numInvalid = 0,
|
|
hasBOM = false,
|
|
confidence = 0;
|
|
const limit = (det.fRawLength / 4) * 4;
|
|
const input = det.fRawInput;
|
|
|
|
if (limit == 0) {
|
|
return null;
|
|
}
|
|
|
|
if (this.getChar(input, 0) == 0x0000feff) {
|
|
hasBOM = true;
|
|
}
|
|
|
|
for (let i = 0; i < limit; i += 4) {
|
|
const ch = this.getChar(input, i);
|
|
|
|
if (ch < 0 || ch >= 0x10ffff || (ch >= 0xd800 && ch <= 0xdfff)) {
|
|
numInvalid += 1;
|
|
} else {
|
|
numValid += 1;
|
|
}
|
|
}
|
|
|
|
// Cook up some sort of confidence score, based on presence of a BOM
|
|
// and the existence of valid and/or invalid multi-byte sequences.
|
|
if (hasBOM && numInvalid == 0) {
|
|
confidence = 100;
|
|
} else if (hasBOM && numValid > numInvalid * 10) {
|
|
confidence = 80;
|
|
} else if (numValid > 3 && numInvalid == 0) {
|
|
confidence = 100;
|
|
} else if (numValid > 0 && numInvalid == 0) {
|
|
confidence = 80;
|
|
} else if (numValid > numInvalid * 10) {
|
|
// Probably corrupt UTF-32BE data. Valid sequences aren't likely by chance.
|
|
confidence = 25;
|
|
}
|
|
|
|
// return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
|
|
return confidence == 0 ? null : match(det, this, confidence);
|
|
}
|
|
}
|
|
|
|
export class UTF_32BE extends UTF_32 {
|
|
name() {
|
|
return 'UTF-32BE';
|
|
}
|
|
getChar(input: Uint8Array, index: number) {
|
|
return (
|
|
((input[index + 0] & 0xff) << 24) |
|
|
((input[index + 1] & 0xff) << 16) |
|
|
((input[index + 2] & 0xff) << 8) |
|
|
(input[index + 3] & 0xff)
|
|
);
|
|
}
|
|
}
|
|
|
|
export class UTF_32LE extends UTF_32 {
|
|
name() {
|
|
return 'UTF-32LE';
|
|
}
|
|
|
|
getChar(input: Uint8Array, index: number) {
|
|
return (
|
|
((input[index + 3] & 0xff) << 24) |
|
|
((input[index + 2] & 0xff) << 16) |
|
|
((input[index + 1] & 0xff) << 8) |
|
|
(input[index + 0] & 0xff)
|
|
);
|
|
}
|
|
}
|