Tesseract  3.02
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
unichar.cpp
Go to the documentation of this file.
1 
2 // File: unichar.cpp
3 // Description: Unicode character/ligature class.
4 // Author: Ray Smith
5 // Created: Wed Jun 28 17:05:01 PDT 2006
6 //
7 // (C) Copyright 2006, Google Inc.
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 // http://www.apache.org/licenses/LICENSE-2.0
12 // Unless required by applicable law or agreed to in writing, software
13 // distributed under the License is distributed on an "AS IS" BASIS,
14 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 // See the License for the specific language governing permissions and
16 // limitations under the License.
17 //
19 
20 #include "unichar.h"
21 
22 #define UNI_MAX_LEGAL_UTF32 0x0010FFFF
23 
24 // Construct from a utf8 string. If len<0 then the string is null terminated.
25 // If the string is too long to fit in the UNICHAR then it takes only what
26 // will fit. Checks for illegal input and stops at an illegal sequence.
27 // The resulting UNICHAR may be empty.
28 UNICHAR::UNICHAR(const char* utf8_str, int len) {
29  int total_len = 0;
30  int step = 0;
31  if (len < 0) {
32  for (len = 0; utf8_str[len] != 0 && len < UNICHAR_LEN; ++len);
33  }
34  for (total_len = 0; total_len < len; total_len += step) {
35  step = utf8_step(utf8_str + total_len);
36  if (total_len + step > UNICHAR_LEN)
37  break; // Too long.
38  if (step == 0)
39  break; // Illegal first byte.
40  int i;
41  for (i = 1; i < step; ++i)
42  if ((utf8_str[total_len + i] & 0xc0) != 0x80)
43  break;
44  if (i < step)
45  break; // Illegal surrogate
46  }
47  memcpy(chars, utf8_str, total_len);
48  if (total_len < UNICHAR_LEN) {
49  chars[UNICHAR_LEN - 1] = total_len;
50  while (total_len < UNICHAR_LEN - 1)
51  chars[total_len++] = 0;
52  }
53 }
54 
55 // Construct from a single UCS4 character. Illegal values are ignored,
56 // resulting in an empty UNICHAR.
57 UNICHAR::UNICHAR(int unicode) {
58  const int bytemask = 0xBF;
59  const int bytemark = 0x80;
60 
61  if (unicode < 0x80) {
62  chars[UNICHAR_LEN - 1] = 1;
63  chars[2] = 0;
64  chars[1] = 0;
65  chars[0] = static_cast<char>(unicode);
66  } else if (unicode < 0x800) {
67  chars[UNICHAR_LEN - 1] = 2;
68  chars[2] = 0;
69  chars[1] = static_cast<char>((unicode | bytemark) & bytemask);
70  unicode >>= 6;
71  chars[0] = static_cast<char>(unicode | 0xc0);
72  } else if (unicode < 0x10000) {
73  chars[UNICHAR_LEN - 1] = 3;
74  chars[2] = static_cast<char>((unicode | bytemark) & bytemask);
75  unicode >>= 6;
76  chars[1] = static_cast<char>((unicode | bytemark) & bytemask);
77  unicode >>= 6;
78  chars[0] = static_cast<char>(unicode | 0xe0);
79  } else if (unicode <= UNI_MAX_LEGAL_UTF32) {
80  chars[UNICHAR_LEN - 1] = 4;
81  chars[3] = static_cast<char>((unicode | bytemark) & bytemask);
82  unicode >>= 6;
83  chars[2] = static_cast<char>((unicode | bytemark) & bytemask);
84  unicode >>= 6;
85  chars[1] = static_cast<char>((unicode | bytemark) & bytemask);
86  unicode >>= 6;
87  chars[0] = static_cast<char>(unicode | 0xf0);
88  } else {
89  memset(chars, 0, UNICHAR_LEN);
90  }
91 }
92 
93 // Get the first character as UCS-4.
94 int UNICHAR::first_uni() const {
95  static const int utf8_offsets[5] = {
96  0, 0, 0x3080, 0xE2080, 0x3C82080
97  };
98  int uni = 0;
99  int len = utf8_step(chars);
100  const char* src = chars;
101 
102  switch (len) {
103  default:
104  break;
105  case 4:
106  uni += static_cast<unsigned char>(*src++);
107  uni <<= 6;
108  case 3:
109  uni += static_cast<unsigned char>(*src++);
110  uni <<= 6;
111  case 2:
112  uni += static_cast<unsigned char>(*src++);
113  uni <<= 6;
114  case 1:
115  uni += static_cast<unsigned char>(*src++);
116  }
117  uni -= utf8_offsets[len];
118  return uni;
119 }
120 
121 // Get a terminated UTF8 string: Must delete[] it after use.
122 char* UNICHAR::utf8_str() const {
123  int len = utf8_len();
124  char* str = new char[len + 1];
125  memcpy(str, chars, len);
126  str[len] = 0;
127  return str;
128 }
129 
130 // Get the number of bytes in the first character of the given utf8 string.
131 int UNICHAR::utf8_step(const char* utf8_str) {
132  static const char utf8_bytes[256] = {
133  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
134  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
135  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
136  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
137  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
138  0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0, 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,
139  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
140  3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4,0,0,0,0,0,0,0,0
141  };
142 
143  return utf8_bytes[static_cast<unsigned char>(*utf8_str)];
144 }