chardet/encoding/unicode.js

var util = require('util'),
    Match = require ('../match');

/**
 * This class matches UTF-16 and UTF-32, both big- and little-endian. The
 * BOM will be used if it is present.
 */

module.exports.UTF_16BE = function() {
    this.name = function() {
        return 'UTF-16BE';
    };
    this.match = function(det) {
        var input = det.fRawInput;

        if (input.length >= 2 && ((input[0] & 0xFF) == 0xFE && (input[1] & 0xFF) == 0xFF)) {
            var confidence = 100;
            return new Match(det, this, confidence);
        }

        // TODO: Do some statistics to check for unsigned UTF-16BE
        return null;
    }
};

module.exports.UTF_16LE = function() {
    this.name = function() {
        return 'UTF-16LE';
    };
    this.match = function(det) {
        var input = det.fRawInput;

        if (input.length >= 2 && ((input[0] & 0xFF) == 0xFF && (input[1] & 0xFF) == 0xFE)) {
           // An LE BOM is present.
           if (input.length >= 4 && input[2] == 0x00 && input[3] == 0x00) {
               // It is probably UTF-32 LE, not UTF-16
               return null;
           }
           var confidence = 100;
           return new Match(det, this, confidence);
        }

        // TODO: Do some statistics to check for unsigned UTF-16LE
        return null;
    }
};

function UTF_32() {};
UTF_32.prototype.match = function(det) {
    var input      = det.fRawInput,
        limit      = (det.fRawLength / 4) * 4,
        numValid   = 0,
        numInvalid = 0,
        hasBOM     = false,
        confidence = 0;

    if (limit==0) {
        return null;
    }

    if (this.getChar(input, 0) == 0x0000FEFF) {
        hasBOM = true;
    }

    for(var i = 0; i < limit; i += 4) {
        var ch = this.getChar(input, i);

        if (ch < 0 || ch >= 0x10FFFF || (ch >= 0xD800 && ch <= 0xDFFF)) {
            numInvalid += 1;
        } else {
            numValid += 1;
        }
    }

    // Cook up some sort of confidence score, based on presence of a BOM
    //    and the existence of valid and/or invalid multi-byte sequences.
    if (hasBOM && numInvalid == 0) {
        confidence = 100;
    } else if (hasBOM && numValid > numInvalid * 10) {
        confidence = 80;
    } else if (numValid > 3 && numInvalid == 0) {
        confidence = 100;
    } else if (numValid > 0 && numInvalid == 0) {
        confidence = 80;
    } else if (numValid > numInvalid*10) {
        // Probably corrupt UTF-32BE data.  Valid sequences aren't likely by chance.
        confidence = 25;
    }

    // return confidence == 0 ? null : new CharsetMatch(det, this, confidence);
    return confidence == 0 ? null : new Match(det, this, confidence);
};

module.exports.UTF_32BE = function() {
    this.name = function() {
        return 'UTF-32BE';
    };
    this.getChar = function(input, index) {
        return (input[index + 0] & 0xFF) << 24 | (input[index + 1] & 0xFF) << 16 |
               (input[index + 2] & 0xFF) <<  8 | (input[index + 3] & 0xFF);
    };
};
util.inherits(module.exports.UTF_32BE, UTF_32);

module.exports.UTF_32LE = function() {
    this.name = function() {
        return 'UTF-32LE';
    };
    this.getChar = function(input, index) {
        return (input[index + 3] & 0xFF) << 24 | (input[index + 2] & 0xFF) << 16 |
               (input[index + 1] & 0xFF) <<  8 | (input[index + 0] & 0xFF);
    };
};
util.inherits(module.exports.UTF_32LE, UTF_32);