builtin: add input_rune/0 and input_rune_iterator/0 + tests (#25238)

.gitattributes

@@ -8,3 +8,4 @@ v.mod linguist-language=V
 .vdocignore linguist-language=ignore
 
 Dockerfile.* linguist-language=Dockerfile
+vlib/v/tests/runes.txt binary

vlib/builtin/input_rune_iterator.v

@@ -0,0 +1,40 @@
+module builtin
+
+// input_rune returns a single rune from the standart input (an unicode codepoint).
+// It expects, that the input is utf8 encoded.
+// It will return `none` on EOF.
+pub fn input_rune() ?rune {
+	x := input_character()
+	if x <= 0 {
+		return none
+	}
+	char_len := utf8_char_len(u8(x))
+	if char_len == 1 {
+		return x
+	}
+	mut b := u8(x)
+	b = b << char_len
+	mut res := rune(b)
+	mut shift := 6 - char_len
+	for i := 1; i < char_len; i++ {
+		c := rune(input_character())
+		res = rune(res) << shift
+		res |= c & 63 // 0x3f
+		shift = 6
+	}
+	return res
+}
+
+// InputRuneIterator is an iterator over the input runes.
+pub struct InputRuneIterator {}
+
+// next returns the next rune from the input stream.
+pub fn (mut self InputRuneIterator) next() ?rune {
+	return input_rune()
+}
+
+// input_rune_iterator returns an iterator to allow for `for i, r in input_rune_iterator() {`.
+// When the input stream is closed, the loop will break.
+pub fn input_rune_iterator() InputRuneIterator {
+	return InputRuneIterator{}
+}

vlib/builtin/input_rune_iterator_test.v

@@ -0,0 +1,51 @@
+// vtest build: !windows
+import os
+import time
+
+fn test_input_rune_iterator_with_unicode_input() {
+	mut p := os.new_process(@VEXE)
+	p.set_args(['-e', 'for i, r in input_rune_iterator() { println("> i: \${i:04} | r: `\${r}`") }'])
+	p.set_redirect_stdio()
+	p.run()
+	spawn fn [mut p] () {
+		time.sleep(10 * time.millisecond)
+		dump(p.pid)
+		p.stdin_write('Проба Abc 你好 🌍 123')
+		time.sleep(10 * time.millisecond)
+		p.stdin_write('\0x00') // 0 should break the input stream
+		time.sleep(10 * time.millisecond)
+		eprintln('>>> done')
+	}()
+	mut olines := []string{}
+	for p.is_alive() {
+		if oline := p.pipe_read(.stdout) {
+			olines << oline
+		}
+		time.sleep(1 * time.millisecond)
+	}
+	p.close()
+	p.wait()
+	assert p.code == 0
+	eprintln('done')
+	solines := olines.join('\n').trim_space().replace('\r', '')
+	eprintln('solines.len: ${solines.len} | solines: ${solines}')
+	assert solines.len > 100
+	assert solines == '> i: 0000 | r: `П`
+> i: 0001 | r: `р`
+> i: 0002 | r: `о`
+> i: 0003 | r: `б`
+> i: 0004 | r: `а`
+> i: 0005 | r: ` `
+> i: 0006 | r: `A`
+> i: 0007 | r: `b`
+> i: 0008 | r: `c`
+> i: 0009 | r: ` `
+> i: 0010 | r: `你`
+> i: 0011 | r: `好`
+> i: 0012 | r: ` `
+> i: 0013 | r: `🌍`
+> i: 0014 | r: ` `
+> i: 0015 | r: `1`
+> i: 0016 | r: `2`
+> i: 0017 | r: `3`'
+}

vlib/v/tests/runes.txt

@@ -0,0 +1 @@
+Проба Abc 你好 🌍 123