1 <?php
2 /**
3 * Tools for conversion between UTF-8 and unicode
4 * The Original Code is Mozilla Communicator client code.
5 * The Initial Developer of the Original Code is
6 * Netscape Communications Corporation.
7 * Portions created by the Initial Developer are Copyright (C) 1998
8 * the Initial Developer. All Rights Reserved.
9 * Ported to PHP by Henri Sivonen (http://hsivonen.iki.fi)
10 * Slight modifications to fit with phputf8 library by Harry Fuecks (hfuecks gmail com)
11 * @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUTF8ToUnicode.cpp
12 * @see http://lxr.mozilla.org/seamonkey/source/intl/uconv/src/nsUnicodeToUTF8.cpp
13 * @see http://hsivonen.iki.fi/php-utf8/
14 * @package utf8
15 */
16
17 //--------------------------------------------------------------------
18 /**
19 * Takes an UTF-8 string and returns an array of ints representing the
20 * Unicode characters. Astral planes are supported ie. the ints in the
21 * output can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates
22 * are not allowed.
23 * Returns false if the input string isn't a valid UTF-8 octet sequence
24 * and raises a PHP error at level E_USER_WARNING
25 * Note: this function has been modified slightly in this library to
26 * trigger errors on encountering bad bytes
27 * @author <hsivonen@iki.fi>
28 * @param string UTF-8 encoded string
29 * @return mixed array of unicode code points or FALSE if UTF-8 invalid
30 * @see utf8_from_unicode
31 * @see http://hsivonen.iki.fi/php-utf8/
32 * @package utf8
33 */
34 function utf8_to_unicode($str) {
35 $mState = 0; // cached expected number of octets after the current octet
36 // until the beginning of the next UTF8 character sequence
37 $mUcs4 = 0; // cached Unicode character
38 $mBytes = 1; // cached expected number of octets in the current sequence
39
40 $out = array();
41
42 $len = strlen($str);
43
44 for($i = 0; $i < $len; $i++) {
45
46 $in = ord($str{$i});
47
48 if ( $mState == 0) {
49
50 // When mState is zero we expect either a US-ASCII character or a
51 // multi-octet sequence.
52 if (0 == (0x80 & ($in))) {
53 // US-ASCII, pass straight through.
54 $out[] = $in;
55 $mBytes = 1;
56
57 } else if (0xC0 == (0xE0 & ($in))) {
58 // First octet of 2 octet sequence
59 $mUcs4 = ($in);
60 $mUcs4 = ($mUcs4 & 0x1F) << 6;
61 $mState = 1;
62 $mBytes = 2;
63
64 } else if (0xE0 == (0xF0 & ($in))) {
65 // First octet of 3 octet sequence
66 $mUcs4 = ($in);
67 $mUcs4 = ($mUcs4 & 0x0F) << 12;
68 $mState = 2;
69 $mBytes = 3;
70
71 } else if (0xF0 == (0xF8 & ($in))) {
72 // First octet of 4 octet sequence
73 $mUcs4 = ($in);
74 $mUcs4 = ($mUcs4 & 0x07) << 18;
75 $mState = 3;
76 $mBytes = 4;
77
78 } else if (0xF8 == (0xFC & ($in))) {
79 /* First octet of 5 octet sequence.
80 *
81 * This is illegal because the encoded codepoint must be either
82 * (a) not the shortest form or
83 * (b) outside the Unicode range of 0-0x10FFFF.
84 * Rather than trying to resynchronize, we will carry on until the end
85 * of the sequence and let the later error handling code catch it.
86 */
87 $mUcs4 = ($in);
88 $mUcs4 = ($mUcs4 & 0x03) << 24;
89 $mState = 4;
90 $mBytes = 5;
91
92 } else if (0xFC == (0xFE & ($in))) {
93 // First octet of 6 octet sequence, see comments for 5 octet sequence.
94 $mUcs4 = ($in);
95 $mUcs4 = ($mUcs4 & 1) << 30;
96 $mState = 5;
97 $mBytes = 6;
98
99 } else {
100 /* Current octet is neither in the US-ASCII range nor a legal first
101 * octet of a multi-octet sequence.
102 */
103 trigger_error(
104 'utf8_to_unicode: Illegal sequence identifier '.
105 'in UTF-8 at byte '.$i,
106 E_USER_WARNING
107 );
108 return FALSE;
109
110 }
111
112 } else {
113
114 // When mState is non-zero, we expect a continuation of the multi-octet
115 // sequence
116 if (0x80 == (0xC0 & ($in))) {
117
118 // Legal continuation.
119 $shift = ($mState - 1) * 6;
120 $tmp = $in;
121 $tmp = ($tmp & 0x0000003F) << $shift;
122 $mUcs4 |= $tmp;
123
124 /**
125 * End of the multi-octet sequence. mUcs4 now contains the final
126 * Unicode codepoint to be output
127 */
128 if (0 == --$mState) {
129
130 /*
131 * Check for illegal sequences and codepoints.
132 */
133 // From Unicode 3.1, non-shortest form is illegal
134 if (((2 == $mBytes) && ($mUcs4 < 0x0080)) ||
135 ((3 == $mBytes) && ($mUcs4 < 0x0800)) ||
136 ((4 == $mBytes) && ($mUcs4 < 0x10000)) ||
137 (4 < $mBytes) ||
138 // From Unicode 3.2, surrogate characters are illegal
139 (($mUcs4 & 0xFFFFF800) == 0xD800) ||
140 // Codepoints outside the Unicode range are illegal
141 ($mUcs4 > 0x10FFFF)) {
142
143 trigger_error(
144 'utf8_to_unicode: Illegal sequence or codepoint '.
145 'in UTF-8 at byte '.$i,
146 E_USER_WARNING
147 );
148
149 return FALSE;
150
151 }
152
153 if (0xFEFF != $mUcs4) {
154 // BOM is legal but we don't want to output it
155 $out[] = $mUcs4;
156 }
157
158 //initialize UTF8 cache
159 $mState = 0;
160 $mUcs4 = 0;
161 $mBytes = 1;
162 }
163
164 } else {
165 /**
166 *((0xC0 & (*in) != 0x80) && (mState != 0))
167 * Incomplete multi-octet sequence.
168 */
169 trigger_error(
170 'utf8_to_unicode: Incomplete multi-octet '.
171 ' sequence in UTF-8 at byte '.$i,
172 E_USER_WARNING
173 );
174
175 return FALSE;
176 }
177 }
178 }
179 return $out;
180 }
181
182 //--------------------------------------------------------------------
183 /**
184 * Takes an array of ints representing the Unicode characters and returns
185 * a UTF-8 string. Astral planes are supported ie. the ints in the
186 * input can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates
187 * are not allowed.
188 * Returns false if the input array contains ints that represent
189 * surrogates or are outside the Unicode range
190 * and raises a PHP error at level E_USER_WARNING
191 * Note: this function has been modified slightly in this library to use
192 * output buffering to concatenate the UTF-8 string (faster) as well as
193 * reference the array by it's keys
194 * @param array of unicode code points representing a string
195 * @return mixed UTF-8 string or FALSE if array contains invalid code points
196 * @author <hsivonen@iki.fi>
197 * @see utf8_to_unicode
198 * @see http://hsivonen.iki.fi/php-utf8/
199 * @package utf8
200 */
201 function utf8_from_unicode($arr) {
202 ob_start();
203
204 foreach (array_keys($arr) as $k) {
205
206 # ASCII range (including control chars)
207 if ( ($arr[$k] >= 0) && ($arr[$k] <= 0x007f) ) {
208
209 echo chr($arr[$k]);
210
211 # 2 byte sequence
212 } else if ($arr[$k] <= 0x07ff) {
213
214 echo chr(0xc0 | ($arr[$k] >> 6));
215 echo chr(0x80 | ($arr[$k] & 0x003f));
216
217 # Byte order mark (skip)
218 } else if($arr[$k] == 0xFEFF) {
219
220 // nop -- zap the BOM
221
222 # Test for illegal surrogates
223 } else if ($arr[$k] >= 0xD800 && $arr[$k] <= 0xDFFF) {
224
225 // found a surrogate
226 trigger_error(
227 'utf8_from_unicode: Illegal surrogate '.
228 'at index: '.$k.', value: '.$arr[$k],
229 E_USER_WARNING
230 );
231
232 return FALSE;
233
234 # 3 byte sequence
235 } else if ($arr[$k] <= 0xffff) {
236
237 echo chr(0xe0 | ($arr[$k] >> 12));
238 echo chr(0x80 | (($arr[$k] >> 6) & 0x003f));
239 echo chr(0x80 | ($arr[$k] & 0x003f));
240
241 # 4 byte sequence
242 } else if ($arr[$k] <= 0x10ffff) {
243
244 echo chr(0xf0 | ($arr[$k] >> 18));
245 echo chr(0x80 | (($arr[$k] >> 12) & 0x3f));
246 echo chr(0x80 | (($arr[$k] >> 6) & 0x3f));
247 echo chr(0x80 | ($arr[$k] & 0x3f));
248
249 } else {
250
251 trigger_error(
252 'utf8_from_unicode: Codepoint out of Unicode range '.
253 'at index: '.$k.', value: '.$arr[$k],
254 E_USER_WARNING
255 );
256
257 // out of range
258 return FALSE;
259 }
260 }
261
262 $result = ob_get_contents();
263 ob_end_clean();
264 return $result;
265 }
266