File size: 6,964 Bytes
1d3f990 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 | /**
* Unicode IR Engine
* Handles Unicode normalization, encoding/decoding, and preservation of astral characters
*/
class UnicodeIREngine {
constructor() {
this.encoder = new TextEncoder();
this.decoder = new TextDecoder('utf-8');
this.normalizationForm = 'NFC'; // Canonical composition
}
/**
* Normalize Unicode string using specified form
*/
normalize(input, form = this.normalizationForm) {
if (typeof input !== 'string') return '';
try {
return input.normalize(form);
} catch (e) {
console.error('Normalization failed:', e);
return input;
}
}
/**
* Encode string to Unicode IR (code points + UTF-8)
* Preserves astral plane, emoji, combining marks, bidirectional text
*/
encode(input) {
const normalized = this.normalize(input);
const codePoints = [];
const utf8Bytes = [];
// Iterate by code point, not by UTF-16 unit
for (const char of normalized) {
const codePoint = char.codePointAt(0);
codePoints.push(codePoint);
}
// Encode to UTF-8
const utf8Array = this.encoder.encode(normalized);
for (let i = 0; i < utf8Array.length; i++) {
utf8Bytes.push(utf8Array[i]);
}
return {
normalized: normalized,
codePoints: codePoints,
utf8Bytes: Array.from(utf8Bytes),
length: codePoints.length,
byteLength: utf8Bytes.length,
};
}
/**
* Decode Unicode IR back to string
* Reverses encode() with full preservation
*/
decode(irObject) {
if (!irObject || !irObject.utf8Bytes) {
console.error('Invalid IR object');
return '';
}
try {
const uint8Array = new Uint8Array(irObject.utf8Bytes);
const decoded = this.decoder.decode(uint8Array);
return this.normalize(decoded);
} catch (e) {
console.error('Decoding failed:', e);
return '';
}
}
/**
* Verify roundtrip preservation (normalize → encode → decode → verify)
*/
verifyRoundtrip(input) {
const encoded = this.encode(input);
const decoded = this.decode(encoded);
const reencoded = this.encode(decoded);
return {
success: encoded.codePoints.length === reencoded.codePoints.length &&
encoded.utf8Bytes.length === reencoded.utf8Bytes.length,
original: input,
encoded: encoded,
decoded: decoded,
reencoded: reencoded,
};
}
/**
* Extract grapheme clusters (visual characters)
* Important for combining marks and emoji sequences
*/
graphemeClusters(input) {
const normalized = this.normalize(input);
const clusters = [];
// Use Intl.Segmenter if available (modern browsers)
if (typeof Intl !== 'undefined' && Intl.Segmenter) {
const segmenter = new Intl.Segmenter('en', { granularity: 'grapheme' });
const segments = segmenter.segment(normalized);
for (const segment of segments) {
clusters.push(segment.segment);
}
} else {
// Fallback: iterate by code point
for (const char of normalized) {
clusters.push(char);
}
}
return clusters;
}
/**
* Detect bidirectional text (RTL/LTR)
*/
detectBidiLevel(input) {
// Simple heuristic: check for RTL scripts (Hebrew, Arabic, etc.)
const rtlRanges = [
[0x0590, 0x08FF], // Hebrew, Arabic, Syriac, etc.
[0xFB1D, 0xFB4F], // Hebrew presentation forms
[0xFB50, 0xFDFF], // Arabic presentation forms A
[0xFE70, 0xFEFF], // Arabic presentation forms B
];
for (const char of input) {
const codePoint = char.codePointAt(0);
for (const [start, end] of rtlRanges) {
if (codePoint >= start && codePoint <= end) {
return 'rtl';
}
}
}
return 'ltr';
}
/**
* Validate astral plane characters
*/
hasAstralCharacters(input) {
for (const char of input) {
if (char.codePointAt(0) > 0xFFFF) {
return true;
}
}
return false;
}
/**
* List all astral characters (code points > 0xFFFF)
*/
findAstralCharacters(input) {
const astral = [];
for (const char of input) {
const codePoint = char.codePointAt(0);
if (codePoint > 0xFFFF) {
astral.push({
char: char,
codePoint: codePoint,
hex: '0x' + codePoint.toString(16).toUpperCase(),
});
}
}
return astral;
}
/**
* Sanitize for safe display (no invisible characters, control chars)
*/
sanitizeForDisplay(input, removeControls = false) {
let result = input;
if (removeControls) {
// Remove control characters (0x0000-0x001F, 0x007F-0x009F)
result = result.replace(/[\x00-\x1F\x7F-\x9F]/g, '');
}
return result;
}
/**
* Check if string contains combining marks
*/
hasCombiningMarks(input) {
// Unicode combining marks range: 0x0300-0x036F
for (const char of input) {
const codePoint = char.codePointAt(0);
if (codePoint >= 0x0300 && codePoint <= 0x036F) {
return true;
}
}
return false;
}
/**
* Export as JSON-safe representation
*/
toJSON(input) {
const ir = this.encode(input);
return {
normalized: ir.normalized,
codePoints: ir.codePoints,
utf8Bytes: ir.utf8Bytes,
metadata: {
length: ir.length,
byteLength: ir.byteLength,
hasAstral: this.hasAstralCharacters(input),
hasCombining: this.hasCombiningMarks(input),
bidiLevel: this.detectBidiLevel(input),
},
};
}
/**
* From JSON-safe representation
*/
fromJSON(obj) {
if (!obj || !obj.utf8Bytes) {
throw new Error('Invalid JSON-safe IR object');
}
return this.decode(obj);
}
}
// Export for use in other modules
if (typeof module !== 'undefined' && module.exports) {
module.exports = UnicodeIREngine;
}
|