diff --git a/.gitattributes b/.gitattributes index 47a6d90d27..ef7f2882b7 100644 --- a/.gitattributes +++ b/.gitattributes @@ -8,3 +8,4 @@ v.mod linguist-language=V .vdocignore linguist-language=ignore Dockerfile.* linguist-language=Dockerfile +vlib/v/tests/runes.txt binary diff --git a/vlib/builtin/input_rune_iterator.v b/vlib/builtin/input_rune_iterator.v new file mode 100644 index 0000000000..3091986879 --- /dev/null +++ b/vlib/builtin/input_rune_iterator.v @@ -0,0 +1,40 @@ +module builtin + +// input_rune returns a single rune from the standart input (an unicode codepoint). +// It expects, that the input is utf8 encoded. +// It will return `none` on EOF. +pub fn input_rune() ?rune { + x := input_character() + if x <= 0 { + return none + } + char_len := utf8_char_len(u8(x)) + if char_len == 1 { + return x + } + mut b := u8(x) + b = b << char_len + mut res := rune(b) + mut shift := 6 - char_len + for i := 1; i < char_len; i++ { + c := rune(input_character()) + res = rune(res) << shift + res |= c & 63 // 0x3f + shift = 6 + } + return res +} + +// InputRuneIterator is an iterator over the input runes. +pub struct InputRuneIterator {} + +// next returns the next rune from the input stream. +pub fn (mut self InputRuneIterator) next() ?rune { + return input_rune() +} + +// input_rune_iterator returns an iterator to allow for `for i, r in input_rune_iterator() {`. +// When the input stream is closed, the loop will break. +pub fn input_rune_iterator() InputRuneIterator { + return InputRuneIterator{} +} diff --git a/vlib/builtin/input_rune_iterator_test.v b/vlib/builtin/input_rune_iterator_test.v new file mode 100644 index 0000000000..f5667b0262 --- /dev/null +++ b/vlib/builtin/input_rune_iterator_test.v @@ -0,0 +1,51 @@ +// vtest build: !windows +import os +import time + +fn test_input_rune_iterator_with_unicode_input() { + mut p := os.new_process(@VEXE) + p.set_args(['-e', 'for i, r in input_rune_iterator() { println("> i: \${i:04} | r: `\${r}`") }']) + p.set_redirect_stdio() + p.run() + spawn fn [mut p] () { + time.sleep(10 * time.millisecond) + dump(p.pid) + p.stdin_write('Проба Abc 你好 🌍 123') + time.sleep(10 * time.millisecond) + p.stdin_write('\0x00') // 0 should break the input stream + time.sleep(10 * time.millisecond) + eprintln('>>> done') + }() + mut olines := []string{} + for p.is_alive() { + if oline := p.pipe_read(.stdout) { + olines << oline + } + time.sleep(1 * time.millisecond) + } + p.close() + p.wait() + assert p.code == 0 + eprintln('done') + solines := olines.join('\n').trim_space().replace('\r', '') + eprintln('solines.len: ${solines.len} | solines: ${solines}') + assert solines.len > 100 + assert solines == '> i: 0000 | r: `П` +> i: 0001 | r: `р` +> i: 0002 | r: `о` +> i: 0003 | r: `б` +> i: 0004 | r: `а` +> i: 0005 | r: ` ` +> i: 0006 | r: `A` +> i: 0007 | r: `b` +> i: 0008 | r: `c` +> i: 0009 | r: ` ` +> i: 0010 | r: `你` +> i: 0011 | r: `好` +> i: 0012 | r: ` ` +> i: 0013 | r: `🌍` +> i: 0014 | r: ` ` +> i: 0015 | r: `1` +> i: 0016 | r: `2` +> i: 0017 | r: `3`' +} diff --git a/vlib/v/tests/runes.txt b/vlib/v/tests/runes.txt new file mode 100644 index 0000000000..6ab946b5e6 --- /dev/null +++ b/vlib/v/tests/runes.txt @@ -0,0 +1 @@ +Проба Abc 你好 🌍 123