chardet/src/encoding/utf8.ts

79 lines
2.0 KiB
TypeScript
Raw Normal View History

import { Context, Recogniser } from '.';
2013-03-03 20:08:00 +00:00
var match = require('../match').default;
2013-03-03 20:08:00 +00:00
export default class Utf8 implements Recogniser {
name() {
2013-11-22 04:37:30 +00:00
return 'UTF-8';
}
2013-03-03 20:08:00 +00:00
match(det: Context) {
2013-11-22 04:37:30 +00:00
var hasBOM = false,
numValid = 0,
numInvalid = 0,
input = det.fRawInput,
trailBytes = 0,
confidence;
2013-03-03 20:08:00 +00:00
if (
det.fRawLength >= 3 &&
(input[0] & 0xff) == 0xef &&
(input[1] & 0xff) == 0xbb &&
(input[2] & 0xff) == 0xbf
) {
2013-11-22 04:37:30 +00:00
hasBOM = true;
}
2013-03-03 20:08:00 +00:00
2013-11-22 04:37:30 +00:00
// Scan for multi-byte sequences
for (var i = 0; i < det.fRawLength; i++) {
var b = input[i];
if ((b & 0x80) == 0) continue; // ASCII
2013-08-16 04:54:08 +00:00
2013-11-22 04:37:30 +00:00
// Hi bit on char found. Figure out how long the sequence should be
if ((b & 0x0e0) == 0x0c0) {
trailBytes = 1;
} else if ((b & 0x0f0) == 0x0e0) {
trailBytes = 2;
} else if ((b & 0x0f8) == 0xf0) {
trailBytes = 3;
} else {
numInvalid++;
if (numInvalid > 5) break;
2013-11-22 04:37:30 +00:00
trailBytes = 0;
}
2013-03-03 20:08:00 +00:00
2013-11-22 04:37:30 +00:00
// Verify that we've got the right number of trail bytes in the sequence
for (;;) {
i++;
if (i >= det.fRawLength) break;
2013-08-21 23:39:36 +00:00
2013-11-22 04:37:30 +00:00
if ((input[i] & 0xc0) != 0x080) {
numInvalid++;
break;
}
if (--trailBytes == 0) {
numValid++;
break;
2013-03-03 20:08:00 +00:00
}
2013-11-22 04:37:30 +00:00
}
}
2013-03-03 20:08:00 +00:00
2013-11-22 04:37:30 +00:00
// Cook up some sort of confidence score, based on presense of a BOM
// and the existence of valid and/or invalid multi-byte sequences.
confidence = 0;
if (hasBOM && numInvalid == 0) confidence = 100;
else if (hasBOM && numValid > numInvalid * 10) confidence = 80;
else if (numValid > 3 && numInvalid == 0) confidence = 100;
else if (numValid > 0 && numInvalid == 0) confidence = 80;
2013-11-22 04:37:30 +00:00
else if (numValid == 0 && numInvalid == 0)
// Plain ASCII.
confidence = 10;
else if (numValid > numInvalid * 10)
// Probably corrupt utf-8 data. Valid sequences aren't likely by chance.
2013-11-22 04:37:30 +00:00
confidence = 25;
else return null;
2013-08-16 04:54:08 +00:00
return match(det, this, confidence);
}
}