AAAAspecials.php000066600000015642151372615370007106 0ustar00 * @param string $string The UTF8 string to strip of special chars * @param string (optional) $repl Replace special with this string * @return string with common non-alphanumeric characters removed * @see utf8_specials_pattern */ function utf8_strip_specials($string, $repl=''){ return preg_replace(utf8_specials_pattern(), $repl, $string); } unicode.php000066600000022164151372615370006726 0ustar00 0xFFFF. Occurrances of the BOM are ignored. Surrogates * are not allowed. * Returns false if the input string isn't a valid UTF-8 octet sequence * and raises a PHP error at level E_USER_WARNING * Note: this function has been modified slightly in this library to * trigger errors on encountering bad bytes * @author * @param string UTF-8 encoded string * @return mixed array of unicode code points or FALSE if UTF-8 invalid * @see utf8_from_unicode * @see http://hsivonen.iki.fi/php-utf8/ * @package utf8 * @subpackage unicode */ function utf8_to_unicode($str) { $mState = 0; // cached expected number of octets after the current octet // until the beginning of the next UTF8 character sequence $mUcs4 = 0; // cached Unicode character $mBytes = 1; // cached expected number of octets in the current sequence $out = array(); $len = strlen($str); for($i = 0; $i < $len; $i++) { $in = ord($str{$i}); if ( $mState == 0) { // When mState is zero we expect either a US-ASCII character or a // multi-octet sequence. if (0 == (0x80 & ($in))) { // US-ASCII, pass straight through. $out[] = $in; $mBytes = 1; } else if (0xC0 == (0xE0 & ($in))) { // First octet of 2 octet sequence $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x1F) << 6; $mState = 1; $mBytes = 2; } else if (0xE0 == (0xF0 & ($in))) { // First octet of 3 octet sequence $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x0F) << 12; $mState = 2; $mBytes = 3; } else if (0xF0 == (0xF8 & ($in))) { // First octet of 4 octet sequence $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x07) << 18; $mState = 3; $mBytes = 4; } else if (0xF8 == (0xFC & ($in))) { /* First octet of 5 octet sequence. * * This is illegal because the encoded codepoint must be either * (a) not the shortest form or * (b) outside the Unicode range of 0-0x10FFFF. * Rather than trying to resynchronize, we will carry on until the end * of the sequence and let the later error handling code catch it. */ $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x03) << 24; $mState = 4; $mBytes = 5; } else if (0xFC == (0xFE & ($in))) { // First octet of 6 octet sequence, see comments for 5 octet sequence. $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 1) << 30; $mState = 5; $mBytes = 6; } else { /* Current octet is neither in the US-ASCII range nor a legal first * octet of a multi-octet sequence. */ trigger_error( 'utf8_to_unicode: Illegal sequence identifier '. 'in UTF-8 at byte '.$i, E_USER_WARNING ); return FALSE; } } else { // When mState is non-zero, we expect a continuation of the multi-octet // sequence if (0x80 == (0xC0 & ($in))) { // Legal continuation. $shift = ($mState - 1) * 6; $tmp = $in; $tmp = ($tmp & 0x0000003F) << $shift; $mUcs4 |= $tmp; /** * End of the multi-octet sequence. mUcs4 now contains the final * Unicode codepoint to be output */ if (0 == --$mState) { /* * Check for illegal sequences and codepoints. */ // From Unicode 3.1, non-shortest form is illegal if (((2 == $mBytes) && ($mUcs4 < 0x0080)) || ((3 == $mBytes) && ($mUcs4 < 0x0800)) || ((4 == $mBytes) && ($mUcs4 < 0x10000)) || (4 < $mBytes) || // From Unicode 3.2, surrogate characters are illegal (($mUcs4 & 0xFFFFF800) == 0xD800) || // Codepoints outside the Unicode range are illegal ($mUcs4 > 0x10FFFF)) { trigger_error( 'utf8_to_unicode: Illegal sequence or codepoint '. 'in UTF-8 at byte '.$i, E_USER_WARNING ); return FALSE; } if (0xFEFF != $mUcs4) { // BOM is legal but we don't want to output it $out[] = $mUcs4; } //initialize UTF8 cache $mState = 0; $mUcs4 = 0; $mBytes = 1; } } else { /** *((0xC0 & (*in) != 0x80) && (mState != 0)) * Incomplete multi-octet sequence. */ trigger_error( 'utf8_to_unicode: Incomplete multi-octet '. ' sequence in UTF-8 at byte '.$i, E_USER_WARNING ); return FALSE; } } } return $out; } //-------------------------------------------------------------------- /** * Takes an array of ints representing the Unicode characters and returns * a UTF-8 string. Astral planes are supported ie. the ints in the * input can be > 0xFFFF. Occurrances of the BOM are ignored. Surrogates * are not allowed. * Returns false if the input array contains ints that represent * surrogates or are outside the Unicode range * and raises a PHP error at level E_USER_WARNING * Note: this function has been modified slightly in this library to use * output buffering to concatenate the UTF-8 string (faster) as well as * reference the array by it's keys * @param array of unicode code points representing a string * @return mixed UTF-8 string or FALSE if array contains invalid code points * @author * @see utf8_to_unicode * @see http://hsivonen.iki.fi/php-utf8/ * @package utf8 * @subpackage unicode */ function utf8_from_unicode($arr) { ob_start(); foreach (array_keys($arr) as $k) { # ASCII range (including control chars) if ( ($arr[$k] >= 0) && ($arr[$k] <= 0x007f) ) { echo chr($arr[$k]); # 2 byte sequence } else if ($arr[$k] <= 0x07ff) { echo chr(0xc0 | ($arr[$k] >> 6)); echo chr(0x80 | ($arr[$k] & 0x003f)); # Byte order mark (skip) } else if($arr[$k] == 0xFEFF) { // nop -- zap the BOM # Test for illegal surrogates } else if ($arr[$k] >= 0xD800 && $arr[$k] <= 0xDFFF) { // found a surrogate trigger_error( 'utf8_from_unicode: Illegal surrogate '. 'at index: '.$k.', value: '.$arr[$k], E_USER_WARNING ); return FALSE; # 3 byte sequence } else if ($arr[$k] <= 0xffff) { echo chr(0xe0 | ($arr[$k] >> 12)); echo chr(0x80 | (($arr[$k] >> 6) & 0x003f)); echo chr(0x80 | ($arr[$k] & 0x003f)); # 4 byte sequence } else if ($arr[$k] <= 0x10ffff) { echo chr(0xf0 | ($arr[$k] >> 18)); echo chr(0x80 | (($arr[$k] >> 12) & 0x3f)); echo chr(0x80 | (($arr[$k] >> 6) & 0x3f)); echo chr(0x80 | ($arr[$k] & 0x3f)); } else { trigger_error( 'utf8_from_unicode: Codepoint out of Unicode range '. 'at index: '.$k.', value: '.$arr[$k], E_USER_WARNING ); // out of range return FALSE; } } $result = ob_get_contents(); ob_end_clean(); return $result; } patterns.php000066600000005656151372615370007147 0ustar00 * @param string string to locate index in * @param int (n times) * @return mixed - int if only one input int, array if more * @return boolean TRUE if it's all ASCII * @package utf8 * @subpackage position */ function utf8_byte_position() { $args = func_get_args(); $str =& array_shift($args); if (!is_string($str)) return false; $result = array(); // trivial byte index, character offset pair $prev = array(0,0); // use a short piece of str to estimate bytes per character // $i (& $j) -> byte indexes into $str $i = utf8_locate_next_chr($str, 300); // $c -> character offset into $str $c = strlen(utf8_decode(substr($str,0,$i))); // deal with arguments from lowest to highest sort($args); foreach ($args as $offset) { // sanity checks FIXME // 0 is an easy check if ($offset == 0) { $result[] = 0; continue; } // ensure no endless looping $safety_valve = 50; do { if ( ($c - $prev[1]) == 0 ) { // Hack: gone past end of string $error = 0; $i = strlen($str); break; } $j = $i + (int)(($offset-$c) * ($i - $prev[0]) / ($c - $prev[1])); // correct to utf8 character boundary $j = utf8_locate_next_chr($str, $j); // save the index, offset for use next iteration $prev = array($i,$c); if ($j > $i) { // determine new character offset $c += strlen(utf8_decode(substr($str,$i,$j-$i))); } else { // ditto $c -= strlen(utf8_decode(substr($str,$j,$i-$j))); } $error = abs($c-$offset); // ready for next time around $i = $j; // from 7 it is faster to iterate over the string } while ( ($error > 7) && --$safety_valve) ; if ($error && $error <= 7) { if ($c < $offset) { // move up while ($error--) { $i = utf8_locate_next_chr($str,++$i); } } else { // move down while ($error--) { $i = utf8_locate_current_chr($str,--$i); } } // ready for next arg $c = $offset; } $result[] = $i; } if ( count($result) == 1 ) { return $result[0]; } return $result; } //-------------------------------------------------------------------- /** * Given a string and any byte index, returns the byte index * of the start of the current UTF-8 character, relative to supplied * position. If the current character begins at the same place as the * supplied byte index, that byte index will be returned. Otherwise * this function will step backwards, looking for the index where * curent UTF-8 character begins * @author Chris Smith * @param string * @param int byte index in the string * @return int byte index of start of next UTF-8 character * @package utf8 * @subpackage position */ function utf8_locate_current_chr( &$str, $idx ) { if ($idx <= 0) return 0; $limit = strlen($str); if ($idx >= $limit) return $limit; // Binary value for any byte after the first in a multi-byte UTF-8 character // will be like 10xxxxxx so & 0xC0 can be used to detect this kind // of byte - assuming well formed UTF-8 while ($idx && ((ord($str[$idx]) & 0xC0) == 0x80)) $idx--; return $idx; } //-------------------------------------------------------------------- /** * Given a string and any byte index, returns the byte index * of the start of the next UTF-8 character, relative to supplied * position. If the next character begins at the same place as the * supplied byte index, that byte index will be returned. * @author Chris Smith * @param string * @param int byte index in the string * @return int byte index of start of next UTF-8 character * @package utf8 * @subpackage position */ function utf8_locate_next_chr( &$str, $idx ) { if ($idx <= 0) return 0; $limit = strlen($str); if ($idx >= $limit) return $limit; // Binary value for any byte after the first in a multi-byte UTF-8 character // will be like 10xxxxxx so & 0xC0 can be used to detect this kind // of byte - assuming well formed UTF-8 while (($idx < $limit) && ((ord($str[$idx]) & 0xC0) == 0x80)) $idx++; return $idx; } validation.php000066600000014752151372615370007436 0ustar00 * @param string UTF-8 encoded string * @return boolean true if valid * @see http://hsivonen.iki.fi/php-utf8/ * @see utf8_compliant * @package utf8 * @subpackage validation */ function utf8_is_valid($str) { $mState = 0; // cached expected number of octets after the current octet // until the beginning of the next UTF8 character sequence $mUcs4 = 0; // cached Unicode character $mBytes = 1; // cached expected number of octets in the current sequence $len = strlen($str); for($i = 0; $i < $len; $i++) { $in = ord($str{$i}); if ( $mState == 0) { // When mState is zero we expect either a US-ASCII character or a // multi-octet sequence. if (0 == (0x80 & ($in))) { // US-ASCII, pass straight through. $mBytes = 1; } else if (0xC0 == (0xE0 & ($in))) { // First octet of 2 octet sequence $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x1F) << 6; $mState = 1; $mBytes = 2; } else if (0xE0 == (0xF0 & ($in))) { // First octet of 3 octet sequence $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x0F) << 12; $mState = 2; $mBytes = 3; } else if (0xF0 == (0xF8 & ($in))) { // First octet of 4 octet sequence $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x07) << 18; $mState = 3; $mBytes = 4; } else if (0xF8 == (0xFC & ($in))) { /* First octet of 5 octet sequence. * * This is illegal because the encoded codepoint must be either * (a) not the shortest form or * (b) outside the Unicode range of 0-0x10FFFF. * Rather than trying to resynchronize, we will carry on until the end * of the sequence and let the later error handling code catch it. */ $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x03) << 24; $mState = 4; $mBytes = 5; } else if (0xFC == (0xFE & ($in))) { // First octet of 6 octet sequence, see comments for 5 octet sequence. $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 1) << 30; $mState = 5; $mBytes = 6; } else { /* Current octet is neither in the US-ASCII range nor a legal first * octet of a multi-octet sequence. */ return FALSE; } } else { // When mState is non-zero, we expect a continuation of the multi-octet // sequence if (0x80 == (0xC0 & ($in))) { // Legal continuation. $shift = ($mState - 1) * 6; $tmp = $in; $tmp = ($tmp & 0x0000003F) << $shift; $mUcs4 |= $tmp; /** * End of the multi-octet sequence. mUcs4 now contains the final * Unicode codepoint to be output */ if (0 == --$mState) { /* * Check for illegal sequences and codepoints. */ // From Unicode 3.1, non-shortest form is illegal if (((2 == $mBytes) && ($mUcs4 < 0x0080)) || ((3 == $mBytes) && ($mUcs4 < 0x0800)) || ((4 == $mBytes) && ($mUcs4 < 0x10000)) || (4 < $mBytes) || // From Unicode 3.2, surrogate characters are illegal (($mUcs4 & 0xFFFFF800) == 0xD800) || // Codepoints outside the Unicode range are illegal ($mUcs4 > 0x10FFFF)) { return FALSE; } //initialize UTF8 cache $mState = 0; $mUcs4 = 0; $mBytes = 1; } } else { /** *((0xC0 & (*in) != 0x80) && (mState != 0)) * Incomplete multi-octet sequence. */ return FALSE; } } } return TRUE; } //-------------------------------------------------------------------- /** * Tests whether a string complies as UTF-8. This will be much * faster than utf8_is_valid but will pass five and six octet * UTF-8 sequences, which are not supported by Unicode and * so cannot be displayed correctly in a browser. In other words * it is not as strict as utf8_is_valid but it's faster. If you use * is to validate user input, you place yourself at the risk that * attackers will be able to inject 5 and 6 byte sequences (which * may or may not be a significant risk, depending on what you are * are doing) * @see utf8_is_valid * @see http://www.php.net/manual/en/reference.pcre.pattern.modifiers.php#54805 * @param string UTF-8 string to check * @return boolean TRUE if string is valid UTF-8 * @package utf8 * @subpackage validation */ function utf8_compliant($str) { if ( strlen($str) == 0 ) { return TRUE; } // If even just the first character can be matched, when the /u // modifier is used, then it's valid UTF-8. If the UTF-8 is somehow // invalid, nothing at all will match, even if the string contains // some valid sequences return (preg_match('/^.{1}/us',$str,$ar) == 1); } ascii.php000066600000020641151372615370006366 0ustar00 * if ( utf8_is_ascii($someString) ) { * // It's just ASCII - use the native PHP version * $someString = strtolower($someString); * } else { * $someString = utf8_strtolower($someString); * } * * * @param string * @return boolean TRUE if it's all ASCII * @package utf8 * @subpackage ascii * @see utf8_is_ascii_ctrl */ function utf8_is_ascii($str) { // Search for any bytes which are outside the ASCII range... return (preg_match('/(?:[^\x00-\x7F])/',$str) !== 1); } //-------------------------------------------------------------------- /** * Tests whether a string contains only 7bit ASCII bytes with device * control codes omitted. The device control codes can be found on the * second table here: http://www.w3schools.com/tags/ref_ascii.asp * * @param string * @return boolean TRUE if it's all ASCII without device control codes * @package utf8 * @subpackage ascii * @see utf8_is_ascii */ function utf8_is_ascii_ctrl($str) { if ( strlen($str) > 0 ) { // Search for any bytes which are outside the ASCII range, // or are device control codes return (preg_match('/[^\x09\x0A\x0D\x20-\x7E]/',$str) !== 1); } return FALSE; } //-------------------------------------------------------------------- /** * Strip out all non-7bit ASCII bytes * If you need to transmit a string to system which you know can only * support 7bit ASCII, you could use this function. * @param string * @return string with non ASCII bytes removed * @package utf8 * @subpackage ascii * @see utf8_strip_non_ascii_ctrl */ function utf8_strip_non_ascii($str) { ob_start(); while ( preg_match( '/^([\x00-\x7F]+)|([^\x00-\x7F]+)/S', $str, $matches) ) { if ( !isset($matches[2]) ) { echo $matches[0]; } $str = substr($str, strlen($matches[0])); } $result = ob_get_contents(); ob_end_clean(); return $result; } //-------------------------------------------------------------------- /** * Strip out device control codes in the ASCII range * which are not permitted in XML. Note that this leaves * multi-byte characters untouched - it only removes device * control codes * @see http://hsivonen.iki.fi/producing-xml/#controlchar * @param string * @return string control codes removed */ function utf8_strip_ascii_ctrl($str) { ob_start(); while ( preg_match( '/^([^\x00-\x08\x0B\x0C\x0E-\x1F\x7F]+)|([\x00-\x08\x0B\x0C\x0E-\x1F\x7F]+)/S', $str, $matches) ) { if ( !isset($matches[2]) ) { echo $matches[0]; } $str = substr($str, strlen($matches[0])); } $result = ob_get_contents(); ob_end_clean(); return $result; } //-------------------------------------------------------------------- /** * Strip out all non 7bit ASCII bytes and ASCII device control codes. * For a list of ASCII device control codes see the 2nd table here: * http://www.w3schools.com/tags/ref_ascii.asp * * @param string * @return boolean TRUE if it's all ASCII * @package utf8 * @subpackage ascii */ function utf8_strip_non_ascii_ctrl($str) { ob_start(); while ( preg_match( '/^([\x09\x0A\x0D\x20-\x7E]+)|([^\x09\x0A\x0D\x20-\x7E]+)/S', $str, $matches) ) { if ( !isset($matches[2]) ) { echo $matches[0]; } $str = substr($str, strlen($matches[0])); } $result = ob_get_contents(); ob_end_clean(); return $result; } //--------------------------------------------------------------- /** * Replace accented UTF-8 characters by unaccented ASCII-7 "equivalents". * The purpose of this function is to replace characters commonly found in Latin * alphabets with something more or less equivalent from the ASCII range. This can * be useful for converting a UTF-8 to something ready for a filename, for example. * Following the use of this function, you would probably also pass the string * through utf8_strip_non_ascii to clean out any other non-ASCII chars * Use the optional parameter to just deaccent lower ($case = -1) or upper ($case = 1) * letters. Default is to deaccent both cases ($case = 0) * * For a more complete implementation of transliteration, see the utf8_to_ascii package * available from the phputf8 project downloads: * http://prdownloads.sourceforge.net/phputf8 * * @param string UTF-8 string * @param int (optional) -1 lowercase only, +1 uppercase only, 1 both cases * @param string UTF-8 with accented characters replaced by ASCII chars * @return string accented chars replaced with ascii equivalents * @author Andreas Gohr * @package utf8 * @subpackage ascii */ function utf8_accents_to_ascii( $str, $case=0 ){ static $UTF8_LOWER_ACCENTS = NULL; static $UTF8_UPPER_ACCENTS = NULL; if($case <= 0){ if ( is_null($UTF8_LOWER_ACCENTS) ) { $UTF8_LOWER_ACCENTS = array( 'à' => 'a', 'ô' => 'o', 'ď' => 'd', 'ḟ' => 'f', 'ë' => 'e', 'š' => 's', 'ơ' => 'o', 'ß' => 'ss', 'ă' => 'a', 'ř' => 'r', 'ț' => 't', 'ň' => 'n', 'ā' => 'a', 'ķ' => 'k', 'ŝ' => 's', 'ỳ' => 'y', 'ņ' => 'n', 'ĺ' => 'l', 'ħ' => 'h', 'ṗ' => 'p', 'ó' => 'o', 'ú' => 'u', 'ě' => 'e', 'é' => 'e', 'ç' => 'c', 'ẁ' => 'w', 'ċ' => 'c', 'õ' => 'o', 'ṡ' => 's', 'ø' => 'o', 'ģ' => 'g', 'ŧ' => 't', 'ș' => 's', 'ė' => 'e', 'ĉ' => 'c', 'ś' => 's', 'î' => 'i', 'ű' => 'u', 'ć' => 'c', 'ę' => 'e', 'ŵ' => 'w', 'ṫ' => 't', 'ū' => 'u', 'č' => 'c', 'ö' => 'oe', 'è' => 'e', 'ŷ' => 'y', 'ą' => 'a', 'ł' => 'l', 'ų' => 'u', 'ů' => 'u', 'ş' => 's', 'ğ' => 'g', 'ļ' => 'l', 'ƒ' => 'f', 'ž' => 'z', 'ẃ' => 'w', 'ḃ' => 'b', 'å' => 'a', 'ì' => 'i', 'ï' => 'i', 'ḋ' => 'd', 'ť' => 't', 'ŗ' => 'r', 'ä' => 'ae', 'í' => 'i', 'ŕ' => 'r', 'ê' => 'e', 'ü' => 'ue', 'ò' => 'o', 'ē' => 'e', 'ñ' => 'n', 'ń' => 'n', 'ĥ' => 'h', 'ĝ' => 'g', 'đ' => 'd', 'ĵ' => 'j', 'ÿ' => 'y', 'ũ' => 'u', 'ŭ' => 'u', 'ư' => 'u', 'ţ' => 't', 'ý' => 'y', 'ő' => 'o', 'â' => 'a', 'ľ' => 'l', 'ẅ' => 'w', 'ż' => 'z', 'ī' => 'i', 'ã' => 'a', 'ġ' => 'g', 'ṁ' => 'm', 'ō' => 'o', 'ĩ' => 'i', 'ù' => 'u', 'į' => 'i', 'ź' => 'z', 'á' => 'a', 'û' => 'u', 'þ' => 'th', 'ð' => 'dh', 'æ' => 'ae', 'µ' => 'u', 'ĕ' => 'e', ); } $str = str_replace( array_keys($UTF8_LOWER_ACCENTS), array_values($UTF8_LOWER_ACCENTS), $str ); } if($case >= 0){ if ( is_null($UTF8_UPPER_ACCENTS) ) { $UTF8_UPPER_ACCENTS = array( 'À' => 'A', 'Ô' => 'O', 'Ď' => 'D', 'Ḟ' => 'F', 'Ë' => 'E', 'Š' => 'S', 'Ơ' => 'O', 'Ă' => 'A', 'Ř' => 'R', 'Ț' => 'T', 'Ň' => 'N', 'Ā' => 'A', 'Ķ' => 'K', 'Ŝ' => 'S', 'Ỳ' => 'Y', 'Ņ' => 'N', 'Ĺ' => 'L', 'Ħ' => 'H', 'Ṗ' => 'P', 'Ó' => 'O', 'Ú' => 'U', 'Ě' => 'E', 'É' => 'E', 'Ç' => 'C', 'Ẁ' => 'W', 'Ċ' => 'C', 'Õ' => 'O', 'Ṡ' => 'S', 'Ø' => 'O', 'Ģ' => 'G', 'Ŧ' => 'T', 'Ș' => 'S', 'Ė' => 'E', 'Ĉ' => 'C', 'Ś' => 'S', 'Î' => 'I', 'Ű' => 'U', 'Ć' => 'C', 'Ę' => 'E', 'Ŵ' => 'W', 'Ṫ' => 'T', 'Ū' => 'U', 'Č' => 'C', 'Ö' => 'Oe', 'È' => 'E', 'Ŷ' => 'Y', 'Ą' => 'A', 'Ł' => 'L', 'Ų' => 'U', 'Ů' => 'U', 'Ş' => 'S', 'Ğ' => 'G', 'Ļ' => 'L', 'Ƒ' => 'F', 'Ž' => 'Z', 'Ẃ' => 'W', 'Ḃ' => 'B', 'Å' => 'A', 'Ì' => 'I', 'Ï' => 'I', 'Ḋ' => 'D', 'Ť' => 'T', 'Ŗ' => 'R', 'Ä' => 'Ae', 'Í' => 'I', 'Ŕ' => 'R', 'Ê' => 'E', 'Ü' => 'Ue', 'Ò' => 'O', 'Ē' => 'E', 'Ñ' => 'N', 'Ń' => 'N', 'Ĥ' => 'H', 'Ĝ' => 'G', 'Đ' => 'D', 'Ĵ' => 'J', 'Ÿ' => 'Y', 'Ũ' => 'U', 'Ŭ' => 'U', 'Ư' => 'U', 'Ţ' => 'T', 'Ý' => 'Y', 'Ő' => 'O', 'Â' => 'A', 'Ľ' => 'L', 'Ẅ' => 'W', 'Ż' => 'Z', 'Ī' => 'I', 'Ã' => 'A', 'Ġ' => 'G', 'Ṁ' => 'M', 'Ō' => 'O', 'Ĩ' => 'I', 'Ù' => 'U', 'Į' => 'I', 'Ź' => 'Z', 'Á' => 'A', 'Û' => 'U', 'Þ' => 'Th', 'Ð' => 'Dh', 'Æ' => 'Ae', 'Ĕ' => 'E', ); } $str = str_replace( array_keys($UTF8_UPPER_ACCENTS), array_values($UTF8_UPPER_ACCENTS), $str ); } return $str; } index.html000066600000000037151372615370006557 0ustar00 bad.php000066600000033324151372615370006026 0ustar00 0 ) { return $badList; } return FALSE; } //-------------------------------------------------------------------- /** * Strips out any bad bytes from a UTF-8 string and returns the rest * PCRE Pattern to locate bad bytes in a UTF-8 string * Comes from W3 FAQ: Multilingual Forms * Note: modified to include full ASCII range including control chars * @see http://www.w3.org/International/questions/qa-forms-utf-8 * @param string * @return string * @package utf8 * @subpackage bad */ function utf8_bad_strip($str) { $UTF8_BAD = '([\x00-\x7F]'. # ASCII (including control chars) '|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte '|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs '|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte '|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates '|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3 '|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15 '|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16 '|(.{1}))'; # invalid byte ob_start(); while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) { if ( !isset($matches[2])) { echo $matches[0]; } $str = substr($str,strlen($matches[0])); } $result = ob_get_contents(); ob_end_clean(); return $result; } //-------------------------------------------------------------------- /** * Replace bad bytes with an alternative character - ASCII character * recommended is replacement char * PCRE Pattern to locate bad bytes in a UTF-8 string * Comes from W3 FAQ: Multilingual Forms * Note: modified to include full ASCII range including control chars * @see http://www.w3.org/International/questions/qa-forms-utf-8 * @param string to search * @param string to replace bad bytes with (defaults to '?') - use ASCII * @return string * @package utf8 * @subpackage bad */ function utf8_bad_replace($str, $replace = '?') { $UTF8_BAD = '([\x00-\x7F]'. # ASCII (including control chars) '|[\xC2-\xDF][\x80-\xBF]'. # non-overlong 2-byte '|\xE0[\xA0-\xBF][\x80-\xBF]'. # excluding overlongs '|[\xE1-\xEC\xEE\xEF][\x80-\xBF]{2}'. # straight 3-byte '|\xED[\x80-\x9F][\x80-\xBF]'. # excluding surrogates '|\xF0[\x90-\xBF][\x80-\xBF]{2}'. # planes 1-3 '|[\xF1-\xF3][\x80-\xBF]{3}'. # planes 4-15 '|\xF4[\x80-\x8F][\x80-\xBF]{2}'. # plane 16 '|(.{1}))'; # invalid byte ob_start(); while (preg_match('/'.$UTF8_BAD.'/S', $str, $matches)) { if ( !isset($matches[2])) { echo $matches[0]; } else { echo $replace; } $str = substr($str,strlen($matches[0])); } $result = ob_get_contents(); ob_end_clean(); return $result; } //-------------------------------------------------------------------- /** * Return code from utf8_bad_identify() when a five octet sequence is detected. * Note: 5 octets sequences are valid UTF-8 but are not supported by Unicode so * do not represent a useful character * @see utf8_bad_identify * @package utf8 * @subpackage bad */ define('UTF8_BAD_5OCTET',1); /** * Return code from utf8_bad_identify() when a six octet sequence is detected. * Note: 6 octets sequences are valid UTF-8 but are not supported by Unicode so * do not represent a useful character * @see utf8_bad_identify * @package utf8 * @subpackage bad */ define('UTF8_BAD_6OCTET',2); /** * Return code from utf8_bad_identify(). * Invalid octet for use as start of multi-byte UTF-8 sequence * @see utf8_bad_identify * @package utf8 * @subpackage bad */ define('UTF8_BAD_SEQID',3); /** * Return code from utf8_bad_identify(). * From Unicode 3.1, non-shortest form is illegal * @see utf8_bad_identify * @package utf8 * @subpackage bad */ define('UTF8_BAD_NONSHORT',4); /** * Return code from utf8_bad_identify(). * From Unicode 3.2, surrogate characters are illegal * @see utf8_bad_identify * @package utf8 * @subpackage bad */ define('UTF8_BAD_SURROGATE',5); /** * Return code from utf8_bad_identify(). * Codepoints outside the Unicode range are illegal * @see utf8_bad_identify * @package utf8 * @subpackage bad */ define('UTF8_BAD_UNIOUTRANGE',6); /** * Return code from utf8_bad_identify(). * Incomplete multi-octet sequence * Note: this is kind of a "catch-all" * @see utf8_bad_identify * @package utf8 * @subpackage bad */ define('UTF8_BAD_SEQINCOMPLETE',7); //-------------------------------------------------------------------- /** * Reports on the type of bad byte found in a UTF-8 string. Returns a * status code on the first bad byte found * @author * @param string UTF-8 encoded string * @return mixed integer constant describing problem or FALSE if valid UTF-8 * @see utf8_bad_explain * @see http://hsivonen.iki.fi/php-utf8/ * @package utf8 * @subpackage bad */ function utf8_bad_identify($str, &$i) { $mState = 0; // cached expected number of octets after the current octet // until the beginning of the next UTF8 character sequence $mUcs4 = 0; // cached Unicode character $mBytes = 1; // cached expected number of octets in the current sequence $len = strlen($str); for($i = 0; $i < $len; $i++) { $in = ord($str{$i}); if ( $mState == 0) { // When mState is zero we expect either a US-ASCII character or a // multi-octet sequence. if (0 == (0x80 & ($in))) { // US-ASCII, pass straight through. $mBytes = 1; } else if (0xC0 == (0xE0 & ($in))) { // First octet of 2 octet sequence $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x1F) << 6; $mState = 1; $mBytes = 2; } else if (0xE0 == (0xF0 & ($in))) { // First octet of 3 octet sequence $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x0F) << 12; $mState = 2; $mBytes = 3; } else if (0xF0 == (0xF8 & ($in))) { // First octet of 4 octet sequence $mUcs4 = ($in); $mUcs4 = ($mUcs4 & 0x07) << 18; $mState = 3; $mBytes = 4; } else if (0xF8 == (0xFC & ($in))) { /* First octet of 5 octet sequence. * * This is illegal because the encoded codepoint must be either * (a) not the shortest form or * (b) outside the Unicode range of 0-0x10FFFF. */ return UTF8_BAD_5OCTET; } else if (0xFC == (0xFE & ($in))) { // First octet of 6 octet sequence, see comments for 5 octet sequence. return UTF8_BAD_6OCTET; } else { // Current octet is neither in the US-ASCII range nor a legal first // octet of a multi-octet sequence. return UTF8_BAD_SEQID; } } else { // When mState is non-zero, we expect a continuation of the multi-octet // sequence if (0x80 == (0xC0 & ($in))) { // Legal continuation. $shift = ($mState - 1) * 6; $tmp = $in; $tmp = ($tmp & 0x0000003F) << $shift; $mUcs4 |= $tmp; /** * End of the multi-octet sequence. mUcs4 now contains the final * Unicode codepoint to be output */ if (0 == --$mState) { // From Unicode 3.1, non-shortest form is illegal if (((2 == $mBytes) && ($mUcs4 < 0x0080)) || ((3 == $mBytes) && ($mUcs4 < 0x0800)) || ((4 == $mBytes) && ($mUcs4 < 0x10000)) ) { return UTF8_BAD_NONSHORT; // From Unicode 3.2, surrogate characters are illegal } else if (($mUcs4 & 0xFFFFF800) == 0xD800) { return UTF8_BAD_SURROGATE; // Codepoints outside the Unicode range are illegal } else if ($mUcs4 > 0x10FFFF) { return UTF8_BAD_UNIOUTRANGE; } //initialize UTF8 cache $mState = 0; $mUcs4 = 0; $mBytes = 1; } } else { // ((0xC0 & (*in) != 0x80) && (mState != 0)) // Incomplete multi-octet sequence. $i--; return UTF8_BAD_SEQINCOMPLETE; } } } if ( $mState != 0 ) { // Incomplete multi-octet sequence. $i--; return UTF8_BAD_SEQINCOMPLETE; } // No bad octets found $i = NULL; return FALSE; } //-------------------------------------------------------------------- /** * Takes a return code from utf8_bad_identify() are returns a message * (in English) explaining what the problem is. * @param int return code from utf8_bad_identify * @return mixed string message or FALSE if return code unknown * @see utf8_bad_identify * @package utf8 * @subpackage bad */ function utf8_bad_explain($code) { switch ($code) { case UTF8_BAD_5OCTET: return 'Five octet sequences are valid UTF-8 but are not supported by Unicode'; break; case UTF8_BAD_6OCTET: return 'Six octet sequences are valid UTF-8 but are not supported by Unicode'; break; case UTF8_BAD_SEQID: return 'Invalid octet for use as start of multi-byte UTF-8 sequence'; break; case UTF8_BAD_NONSHORT: return 'From Unicode 3.1, non-shortest form is illegal'; break; case UTF8_BAD_SURROGATE: return 'From Unicode 3.2, surrogate characters are illegal'; break; case UTF8_BAD_UNIOUTRANGE: return 'Codepoints outside the Unicode range are illegal'; break; case UTF8_BAD_SEQINCOMPLETE: return 'Incomplete multi-octet sequence'; break; } trigger_error('Unknown error code: '.$code,E_USER_WARNING); return FALSE; } .htaccess000066600000000177151372615370006365 0ustar00 Order allow,deny Deny from all