elektrine-haraka/lib/text-normalizer.js

/**
 * Header text normalization helpers.
 *
 * Fixes common mojibake where UTF-8 bytes were interpreted as Latin-1.
 */

'use strict';

function count_replacement_chars(value) {
    if (typeof value !== 'string' || value.length === 0) return 0;
    return (value.match(/\uFFFD/g) || []).length;
}

function has_c1_controls(value) {
    return /[\u0080-\u009F]/.test(value);
}

function count_mojibake_utf8_latin1_pairs(value) {
    let count = 0;
    for (let i = 0; i < value.length - 1; i += 1) {
        const a = value.charCodeAt(i);
        const b = value.charCodeAt(i + 1);
        // Pattern of UTF-8 byte sequences misread as Latin-1 code points.
        if (a >= 0x00C2 && a <= 0x00F4 && b >= 0x0080 && b <= 0x00BF) {
            count += 1;
        }
    }
    return count;
}

function try_repair_utf8_latin1_mojibake(value) {
    if (typeof value !== 'string' || value.length === 0) return value;

    const cps = Array.from(value, (char) => char.codePointAt(0));
    if (cps.some((cp) => cp > 0xFF)) {
        return value;
    }

    const repaired = Buffer.from(cps).toString('utf8');
    if (!repaired) return value;

    const before_controls = has_c1_controls(value);
    const after_controls = has_c1_controls(repaired);
    const before_pairs = count_mojibake_utf8_latin1_pairs(value);
    const after_pairs = count_mojibake_utf8_latin1_pairs(repaired);
    const replacement_count = count_replacement_chars(repaired);

    if (before_controls && !after_controls) {
        if (replacement_count === 0) return repaired;

        // Allow partial recovery when controls disappear and mojibake pairs drop sharply.
        // This handles cases where one continuation byte was dropped upstream.
        if (before_pairs >= 3 && after_pairs < before_pairs && replacement_count <= 2) {
            return repaired;
        }
    }

    if (before_pairs > 0 && after_pairs < before_pairs && replacement_count === 0) {
        return repaired;
    }

    return value;
}

function normalize_header(value) {
    if (typeof value !== 'string') return value;
    return try_repair_utf8_latin1_mojibake(value);
}

module.exports = {
    normalize_header,
    try_repair_utf8_latin1_mojibake
};