From c4ee940047df046a47dd92b6049ae4cebf204de0 Mon Sep 17 00:00:00 2001
From: Turiiya <34311583+ttytm@users.noreply.github.com>
Date: Tue, 10 Oct 2023 19:37:21 +0200
Subject: [PATCH] tests: fix input tests with `expect`, not covering false
 negatives (#19518)

---
 .github/workflows/linux_ci.yml                |  8 ++----
 .github/workflows/macos_ci.yml                |  8 ++----
 examples/password/correct.expect              |  8 ------
 examples/password/incorrect.expect            |  8 ------
 examples/password/password_ci.vsh             |  3 ---
 examples/password/password_test.v             | 27 +++++++++++++++++++
 examples/password/tests/correct.expect        | 12 +++++++++
 examples/password/tests/incorrect.expect      | 12 +++++++++
 .../tests/output_from_expect_arg.expect       | 12 +++++++++
 examples/readline/correct.expect              |  9 -------
 .../readline/{readline_ci.v => readline.v}    |  3 ++-
 examples/readline/readline.vsh                |  2 --
 examples/readline/readline_test.v             | 26 ++++++++++++++++++
 examples/readline/tests/readline.expect       | 15 +++++++++++
 .../tests/readline_from_expect_arg.expect     | 14 ++++++++++
 15 files changed, 124 insertions(+), 43 deletions(-)
 delete mode 100755 examples/password/correct.expect
 delete mode 100755 examples/password/incorrect.expect
 delete mode 100644 examples/password/password_ci.vsh
 create mode 100644 examples/password/password_test.v
 create mode 100755 examples/password/tests/correct.expect
 create mode 100755 examples/password/tests/incorrect.expect
 create mode 100755 examples/password/tests/output_from_expect_arg.expect
 delete mode 100755 examples/readline/correct.expect
 rename examples/readline/{readline_ci.v => readline.v} (64%)
 delete mode 100644 examples/readline/readline.vsh
 create mode 100644 examples/readline/readline_test.v
 create mode 100755 examples/readline/tests/readline.expect
 create mode 100755 examples/readline/tests/readline_from_expect_arg.expect

diff --git a/.github/workflows/linux_ci.yml b/.github/workflows/linux_ci.yml
index 362a251454..7a332b2686 100644
--- a/.github/workflows/linux_ci.yml
+++ b/.github/workflows/linux_ci.yml
@@ -99,13 +99,9 @@ jobs:
           ./v3 version
           ./v3 -o tetris  -usecache examples/tetris/tetris.v
       - name: Test password input
-        run: |
-          ./v examples/password
-          ./v examples/password/password_ci.vsh
+        run: ./v test examples/password/
       - name: Test readline
-        run: |
-          ./v examples/readline/readline_ci.v
-          ./v examples/readline/readline.vsh
+        run: ./v test examples/readline/
 
   ubuntu-tcc-boehm-gc:
     runs-on: ubuntu-20.04
diff --git a/.github/workflows/macos_ci.yml b/.github/workflows/macos_ci.yml
index 778cda84f0..322c6789aa 100644
--- a/.github/workflows/macos_ci.yml
+++ b/.github/workflows/macos_ci.yml
@@ -113,10 +113,6 @@ jobs:
         run: |
           ./v -o v2 -parallel-cc cmd/v
       - name: Test password input
-        run: |
-          ./v examples/password
-          ./v examples/password/password_ci.vsh
+        run: ./v test examples/password/
       - name: Test readline
-        run: |
-          ./v examples/readline/readline_ci.v
-          ./v examples/readline/readline.vsh
+        run: ./v test examples/readline/
diff --git a/examples/password/correct.expect b/examples/password/correct.expect
deleted file mode 100755
index 1ab9c47060..0000000000
--- a/examples/password/correct.expect
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/expect
-spawn ./password
-expect "Enter your password : "
-send "Sample\r"
-expect "Confirm password : "
-send "Sample\r"
-expect "Password confirmed! You entered: Sample ."
-expect eof
diff --git a/examples/password/incorrect.expect b/examples/password/incorrect.expect
deleted file mode 100755
index 61bf9e9247..0000000000
--- a/examples/password/incorrect.expect
+++ /dev/null
@@ -1,8 +0,0 @@
-#!/usr/bin/expect
-spawn ./password
-expect "Enter your password : "
-send "Sample123\r"
-expect "Confirm password : "
-send "Sample234\r"
-expect "Passwords do not match ."
-expect eof
diff --git a/examples/password/password_ci.vsh b/examples/password/password_ci.vsh
deleted file mode 100644
index 2f08c4a18f..0000000000
--- a/examples/password/password_ci.vsh
+++ /dev/null
@@ -1,3 +0,0 @@
-chdir('examples/password')!
-assert execute('./correct.expect').exit_code == 0
-assert execute('./incorrect.expect').exit_code == 0
diff --git a/examples/password/password_test.v b/examples/password/password_test.v
new file mode 100644
index 0000000000..8ced743ca2
--- /dev/null
+++ b/examples/password/password_test.v
@@ -0,0 +1,27 @@
+import os
+
+const (
+	// Expect has to be installed for the test.
+	expect_exe = os.find_abs_path_of_executable('expect') or {
+		eprintln('skipping test, since expect is missing')
+		exit(0)
+	}
+	// Directory that contains the Expect scripts used in the test.
+	expect_tests_path = os.join_path(@VMODROOT, 'examples', 'password', 'tests')
+)
+
+fn test_password_input() {
+	correct := os.execute(os.join_path(expect_tests_path, 'correct.expect'))
+	assert correct.exit_code == 0, correct.output
+
+	incorrect := os.execute(os.join_path(expect_tests_path, 'incorrect.expect'))
+	assert incorrect.exit_code == 0, incorrect.output
+
+	expected_out := 'Enter your password : '
+	mut res := os.execute('${os.join_path(expect_tests_path, 'output_from_expect_arg.expect')} "${expected_out}"')
+	assert res.exit_code == 0, res.output
+
+	not_exptectd_out := 'Enter your passwords : '
+	res = os.execute('${os.join_path(expect_tests_path, 'output_from_expect_arg.expect')} "${not_exptectd_out}"')
+	assert res.exit_code == 1, res.output
+}
diff --git a/examples/password/tests/correct.expect b/examples/password/tests/correct.expect
new file mode 100755
index 0000000000..81139bc32d
--- /dev/null
+++ b/examples/password/tests/correct.expect
@@ -0,0 +1,12 @@
+#!/usr/bin/expect
+
+set timeout 3
+set v_root [exec sh -c "git rev-parse --show-toplevel"]
+
+spawn $v_root/v run $v_root/examples/password/password.v
+
+expect "Enter your password : " { send "Sample\r" } timeout { exit 1 }
+expect "Confirm password : " { send "Sample\r" } timeout { exit 1 }
+expect "Password confirmed! You entered: Sample ." {}  timeout { exit 1 }
+
+expect eof
diff --git a/examples/password/tests/incorrect.expect b/examples/password/tests/incorrect.expect
new file mode 100755
index 0000000000..3d95feaa7e
--- /dev/null
+++ b/examples/password/tests/incorrect.expect
@@ -0,0 +1,12 @@
+#!/usr/bin/expect
+
+set timeout 3
+set v_root [exec sh -c "git rev-parse --show-toplevel"]
+
+spawn $v_root/v run $v_root/examples/password/password.v
+
+expect "Enter your password : " { send "Sample123\r" } timeout { exit 1 }
+expect "Confirm password : " { send "Sample234\r" } timeout { exit 1 }
+expect "Passwords do not match ." {} timeout { exit 1 }
+
+expect eof
diff --git a/examples/password/tests/output_from_expect_arg.expect b/examples/password/tests/output_from_expect_arg.expect
new file mode 100755
index 0000000000..12ae1650b4
--- /dev/null
+++ b/examples/password/tests/output_from_expect_arg.expect
@@ -0,0 +1,12 @@
+#!/usr/bin/expect
+
+set timeout 3
+set v_root [exec sh -c "git rev-parse --show-toplevel"]
+# Send expected output as arg to re-use the script for testing incorrect values.
+set expect_ [lindex $argv 0]
+
+spawn $v_root/v run $v_root/examples/password/password.v
+
+expect $expect_ {} timeout {  exit 1 }
+
+expect eof
diff --git a/examples/readline/correct.expect b/examples/readline/correct.expect
deleted file mode 100755
index 2381e8406b..0000000000
--- a/examples/readline/correct.expect
+++ /dev/null
@@ -1,9 +0,0 @@
-#!/usr/bin/expect
-spawn ./readline_ci
-send "a"
-expect "got 97"
-send "1"
-expect "got 49"
-send "q"
-expect "Goodbye."
-expect eof
diff --git a/examples/readline/readline_ci.v b/examples/readline/readline.v
similarity index 64%
rename from examples/readline/readline_ci.v
rename to examples/readline/readline.v
index 76ab34f5ea..d4a1ffbd21 100644
--- a/examples/readline/readline_ci.v
+++ b/examples/readline/readline.v
@@ -8,9 +8,10 @@ fn main() {
 
 fn run() ! {
 	$if windows {
-		eprintln('skipping test on windows for now')
+		eprintln('skipping on Windows, since raw mode and read_char are not yet implemented.')
 		return
 	} $else {
+		// Explicit comptime block for other OSes than Windows is required to not break compilation on Windows.
 		mut r := readline.Readline{}
 		r.enable_raw_mode_nosig()
 		defer {
diff --git a/examples/readline/readline.vsh b/examples/readline/readline.vsh
deleted file mode 100644
index 8368bcd995..0000000000
--- a/examples/readline/readline.vsh
+++ /dev/null
@@ -1,2 +0,0 @@
-chdir('examples/readline')!
-assert execute('./correct.expect').exit_code == 0
diff --git a/examples/readline/readline_test.v b/examples/readline/readline_test.v
new file mode 100644
index 0000000000..17514ca3b5
--- /dev/null
+++ b/examples/readline/readline_test.v
@@ -0,0 +1,26 @@
+import os
+
+const (
+	// Expect has to be installed for the test.
+	expect_exe = os.find_abs_path_of_executable('expect') or {
+		eprintln('skipping test, since expect is missing')
+		exit(0)
+	}
+	// Directory that contains the Expect scripts used in the test.
+	expect_tests_path = os.join_path(@VMODROOT, 'examples', 'readline', 'tests')
+)
+
+fn test_password_input() {
+	correct := os.execute(os.join_path(expect_tests_path, 'readline.expect'))
+	assert correct.exit_code == 0, correct.output
+
+	send_a := 'a'
+	expect_a := 'got 97' // readline output for `a`
+	a_res := os.execute('${os.join_path(expect_tests_path, 'readline_from_expect_arg.expect')} ${send_a} "${expect_a}"')
+	assert a_res.exit_code == 0, a_res.output
+
+	send_b := 'b'
+	b_res := os.execute('${os.join_path(expect_tests_path, 'readline_from_expect_arg.expect')} ${send_b} "${expect_a}"')
+	assert b_res.exit_code == 1, b_res.output
+	assert b_res.output.contains('got 98')
+}
diff --git a/examples/readline/tests/readline.expect b/examples/readline/tests/readline.expect
new file mode 100755
index 0000000000..564c8645e3
--- /dev/null
+++ b/examples/readline/tests/readline.expect
@@ -0,0 +1,15 @@
+#!/usr/bin/expect
+
+set timeout 3
+set v_root [exec sh -c "git rev-parse --show-toplevel"]
+
+spawn $v_root/v run $v_root/examples/readline/readline.v
+
+send "a"
+expect "got 97" {} timeout { exit 1 }
+send "1"
+expect "got 49" {} timeout { exit 1 }
+send "q"
+expect "Goodbye." {} timeout { exit 1 }
+
+expect eof
diff --git a/examples/readline/tests/readline_from_expect_arg.expect b/examples/readline/tests/readline_from_expect_arg.expect
new file mode 100755
index 0000000000..7b478fc81d
--- /dev/null
+++ b/examples/readline/tests/readline_from_expect_arg.expect
@@ -0,0 +1,14 @@
+#!/usr/bin/expect
+
+set timeout 3
+set v_root [exec sh -c "git rev-parse --show-toplevel"]
+# Use input arguments for send and expect.
+set send_ [lindex $argv 0]
+set expect_ [lindex $argv 1]
+
+spawn $v_root/v run $v_root/examples/readline/readline.v
+
+send $send_
+expect $expect_ {} timeout { exit 1 }
+
+expect eof