miduo_client/Assets/ManagedResources/~Lua/misc/utf8.lua

307 lines
8.2 KiB
Lua
Raw Blame History

local utf8 = {}
--byte index of the next char after the char at byte index i, followed by a valid flag for the char at byte index i.
--nil if not found. invalid characters are iterated as 1-byte chars.
function utf8.next_raw(s, i)
if not i then
if #s == 0 then return nil end
return 1, true --fake flag (doesn't matter since this flag is not to be taken as full validation)
end
if i > #s then return end
local c = s:byte(i)
if c >= 0x00 and c <= 0x7F then
i = i + 1
elseif c >= 0xC2 and c <= 0xDF then
i = i + 2
elseif c >= 0xE0 and c <= 0xEF then
i = i + 3
elseif c >= 0xF0 and c <= 0xF4 then
i = i + 4
else --invalid
return i + 1, false
end
if i > #s then return end
return i, true
end
--next() is the generic iterator and can be replaced for different semantics. next_raw() must preserve its semantics.
utf8.next = utf8.next_raw
--iterate chars, returning the byte index where each char starts
function utf8.byte_indices(s, previ)
return utf8.next, s, previ
end
--number of chars in string
function utf8.len(s)
assert(s, "bad argument #1 to 'len' (string expected, got nil)")
local len = 0
for _ in utf8.byte_indices(s) do
len = len + 1
end
return len
end
--byte index given char index. nil if the index is outside the string.
function utf8.byte_index(s, target_ci)
if target_ci < 1 then return end
local ci = 0
for i in utf8.byte_indices(s) do
ci = ci + 1
if ci == target_ci then
return i
end
end
assert(target_ci > ci, "invalid index")
end
--char index given byte index. nil if the index is outside the string.
function utf8.char_index(s, target_i)
if target_i < 1 or target_i > #s then return end
local ci = 0
for i in utf8.byte_indices(s) do
ci = ci + 1
if i == target_i then
return ci
end
end
error("invalid index")
end
--byte index of the prev. char before the char at byte index i, which defaults to #s + 1.
--nil if the index is outside the 2..#s+1 range.
--NOTE: unlike next(), this is a O(N) operation!
function utf8.prev(s, nexti)
nexti = nexti or #s + 1
if nexti <= 1 or nexti > #s + 1 then return end
local lasti, lastvalid = utf8.next(s)
for i, valid in utf8.byte_indices(s) do
if i == nexti then
return lasti, lastvalid
end
lasti, lastvalid = i, valid
end
if nexti == #s + 1 then
return lasti, lastvalid
end
error("invalid index")
end
--iterate chars in reverse order, returning the byte index where each char starts.
function utf8.byte_indices_reverse(s, nexti)
if #s < 200 then
--using prev() is a O(N^2/2) operation, ok for small strings (200 chars need 40,000 iterations)
return utf8.prev, s, nexti
else
--store byte indices in a table and iterate them in reverse.
--this is 40x slower than byte_indices() but still fast at 2mil chars/second (but eats RAM and makes garbage).
local t = {}
for i in utf8.byte_indices(s) do
if nexti and i >= nexti then break end
table.insert(t, i)
end
local i = #t + 1
return function()
i = i - 1
return t[i]
end
end
end
--sub based on char indices, which, unlike with standard string.sub(), can't be negative.
--start_ci can be 1..inf and end_ci can be 0..inf. end_ci can be nil meaning last char.
--if start_ci is out of range or end_ci < start_ci, the empty string is returned.
--if end_ci is out of range, it is considered to be the last position in the string.
function utf8.sub(s, start_ci, end_ci)
--assert for positive indices because we might implement negative indices in the future.
assert(start_ci >= 1)
assert(not end_ci or end_ci >= 0)
local ci = 0
local start_i, end_i
for i in utf8.byte_indices(s) do
ci = ci + 1
if ci == start_ci then
start_i = i
end
if ci == end_ci then
end_i = i
end
end
if not start_i then
assert(start_ci > ci, 'invalid index')
return ''
end
if end_ci and not end_i then
if end_ci < start_ci then
return ''
end
assert(end_ci > ci, 'invalid index')
end
return s:sub(start_i, end_i and end_i - 1)
end
--check if a string contains a substring at byte index i without making garbage.
--nil if the index is out of range. true if searching for the empty string.
function utf8.contains(s, i, sub)
if i < 1 or i > #s then return nil end
for si = 1, #sub do
if s:byte(i + si - 1) ~= sub:byte(si) then
return false
end
end
return true
end
--count the number of occurences of a substring in a string. the substring cannot be the empty string.
function utf8.count(s, sub)
assert(#sub > 0)
local count = 0
local i = 1
while i do
if utf8.contains(s, i, sub) then
count = count + 1
i = i + #sub
if i > #s then break end
else
i = utf8.next(s, i)
end
end
return count
end
--utf8 validation and sanitization
--check if there's a valid utf8 codepoint at byte index i. valid ranges for each utf8 byte are:
-- byte 1 2 3 4
--------------------------------------------
-- 00 - 7F
-- C2 - DF 80 - BF
-- E0 A0 - BF 80 - BF
-- E1 - EC 80 - BF 80 - BF
-- ED 80 - 9F 80 - BF
-- EE - EF 80 - BF 80 - BF
-- F0 90 - BF 80 - BF 80 - BF
-- F1 - F3 80 - BF 80 - BF 80 - BF
-- F4 80 - 8F 80 - BF 80 - BF
function utf8.isvalid(s, i)
local c = s:byte(i)
if not c then
return false
elseif c >= 0x00 and c <= 0x7F then
return true
elseif c >= 0xC2 and c <= 0xDF then
local c2 = s:byte(i + 1)
return c2 and c2 >= 0x80 and c2 <= 0xBF
elseif c >= 0xE0 and c <= 0xEF then
local c2 = s:byte(i + 1)
local c3 = s:byte(i + 2)
if c == 0xE0 then
return c2 and c3 and
c2 >= 0xA0 and c2 <= 0xBF and
c3 >= 0x80 and c3 <= 0xBF
elseif c >= 0xE1 and c <= 0xEC then
return c2 and c3 and
c2 >= 0x80 and c2 <= 0xBF and
c3 >= 0x80 and c3 <= 0xBF
elseif c == 0xED then
return c2 and c3 and
c2 >= 0x80 and c2 <= 0x9F and
c3 >= 0x80 and c3 <= 0xBF
elseif c >= 0xEE and c <= 0xEF then
if c == 0xEF and c2 == 0xBF and (c3 == 0xBE or c3 == 0xBF) then
return false --uFFFE and uFFFF non-characters
end
return c2 and c3 and
c2 >= 0x80 and c2 <= 0xBF and
c3 >= 0x80 and c3 <= 0xBF
end
elseif c >= 0xF0 and c <= 0xF4 then
local c2 = s:byte(i + 1)
local c3 = s:byte(i + 2)
local c4 = s:byte(i + 3)
if c == 0xF0 then
return c2 and c3 and c4 and
c2 >= 0x90 and c2 <= 0xBF and
c3 >= 0x80 and c3 <= 0xBF and
c4 >= 0x80 and c4 <= 0xBF
elseif c >= 0xF1 and c <= 0xF3 then
return c2 and c3 and c4 and
c2 >= 0x80 and c2 <= 0xBF and
c3 >= 0x80 and c3 <= 0xBF and
c4 >= 0x80 and c4 <= 0xBF
elseif c == 0xF4 then
return c2 and c3 and c4 and
c2 >= 0x80 and c2 <= 0x8F and
c3 >= 0x80 and c3 <= 0xBF and
c4 >= 0x80 and c4 <= 0xBF
end
end
return false
end
--byte index of the next valid utf8 char after the char at byte index i.
--nil if indices go out of range. invalid characters are skipped.
function utf8.next_valid(s, i)
local valid
i, valid = utf8.next_raw(s, i)
while i and (not valid or not utf8.isvalid(s, i)) do
i, valid = utf8.next(s, i)
end
return i
end
--iterate valid chars, returning the byte index where each char starts
function utf8.valid_byte_indices(s)
return utf8.next_valid, s
end
--assert that a string only contains valid utf8 characters
function utf8.validate(s)
for i, valid in utf8.byte_indices(s) do
if not valid or not utf8.isvalid(s, i) then
error(string.format('invalid utf8 char at #%d', i))
end
end
end
local function table_lookup(s, i, j, t)
return t[s:sub(i, j)]
end
--replace characters in string based on a function f(s, i, j, ...) -> replacement_string | nil
function utf8.replace(s, f, ...)
if type(f) == 'table' then
return utf8.replace(s, table_lookup, f)
end
if s == '' then
return s
end
local t = {}
local lasti = 1
for i in utf8.byte_indices(s) do
local nexti = utf8.next(s, i) or #s + 1
local repl = f(s, i, nexti - 1, ...)
if repl then
table.insert(t, s:sub(lasti, i - 1))
table.insert(t, repl)
lasti = nexti
end
end
table.insert(t, s:sub(lasti))
return table.concat(t)
end
local function replace_invalid(s, i, j, repl_char)
if not utf8.isvalid(s, i) then
return repl_char
end
end
--replace invalid utf8 chars with a replacement char
function utf8.sanitize(s, repl_char)
repl_char = repl_char or '<EFBFBD>' --\uFFFD
return utf8.replace(s, replace_invalid, repl_char)
end
return utf8