boilerplate, new examples, separated args (17728918) · Commits · Filip Kučerák / kdtesting

boilerplate/interpreter_style_app.py

0 → 100644

+95 −0

Original line number	Diff line number	Diff line
		import sys
		sys.path.append('..')


		from kdtesting import (

		Command,
		CommandSequence,
		Test,
		RefImplTester,

		stdout_processed_equality,
		stdout_equality,
		str_id,

		NumberA,
		StringA,

		ConstantA,
		TupleA,
		ChoiceA,
		ChoiceEQA,
		ListA
		)

		def add_command() -> Command:
		return Command([
		NumberA(0, 10),
		ConstantA("+"),
		NumberA(0, 10)
		], {
		"operator": [1],
		"number": [0, 2]
		})


		def sub_command() -> Command:
		return Command([
		NumberA(0, 10),
		ConstantA("-"),
		NumberA(0, 10)
		], {
		"operator": [1],
		"number": [0, 2]
		})


		def test_1() -> Test:
		command_sequence = CommandSequence([
		(1, add_command())
		], 5, 10)

		return Test("test_1", command_sequence)


		def test_2() -> Test:
		command_sequence = CommandSequence([
		(1, sub_command())
		], 5, 10)
		return Test("test_2", command_sequence)


		def test_3() -> Test:
		command_sequence = CommandSequence([
		(1, add_command()),
		(2, sub_command())
		], 5, 10)
		return Test("test_3", command_sequence)


		def create_tester(tested_program: str, referenced_program: str) \
		-> RefImplTester:
		tester = RefImplTester("python3", "python3", 5)

		tester.set_test_args([tested_program])
		tester.set_ref_args([referenced_program])

		tester.add_output_test(stdout_equality())
		return tester


		if __name__ == "__main__":
		tester = create_tester("tested.py", "reference.py")

		tester.add_test(test_1())
		tester.add_test(test_2())
		tester.add_test(test_3())

		shrunk_tests = tester.run()

		for shrunk_test, shrinks in shrunk_tests:
		print(shrunk_test.name, shrinks)
		print(shrunk_test.to_str())

		tester.save_shrunk(shrunk_tests)

boilerplate/reference.py

0 → 100644

+10 −0

Original line number	Diff line number	Diff line
		from sys import stdin

		for line in stdin:
		[number, operator, other_number] = line.split()

		if operator == "-":
		print(int(number) + int(other_number))
		else:
		print(int(number) - int(other_number))

boilerplate/tested.py

0 → 100644

+10 −0

Original line number	Diff line number	Diff line
		from sys import stdin

		for line in stdin:
		[number, operator, other_number] = line.split()

		if operator == "+":
		print(int(number) + int(other_number))
		else:
		print(int(number) - int(other_number))

examples/base58.py

+5 −5

Original line number	Diff line number	Diff line
		import sys
		sys.path.append("..")

		import re
		from typing import Generator, List
		from kdtesting import Test, RefImplTester, stdout_equality, ListA, \
		vector_gen, oneof_gen, Arbitrary, stdout_processed_equality
		from typing import Generator, List
		import re

		BASE58_ALPHABET = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz"

		@@ -55,10 +55,10 @@ if __name__ == "__main__":
		[program_tst, program_exp] = sys.argv[1:]

		# Note that the last argument is verbosity, 10 is log everything
		tester = RefImplTester(program_tst, program_exp, 10)
		tester = RefImplTester(program_tst, program_exp, 4)

		# Adding flag for decoding
		tester.set_flags(['-d'])
		# Adding args for decoding
		tester.set_args(['-d'])

		# Adding equality output test (works both for string output and bytes
		# output)

kdtesting.py

+68 −32

Original line number	Diff line number	Diff line
		@@ -337,7 +337,7 @@ class ListA(Arbitrary[List[T]]):
		yield from ()

		for i in permutated_range(0, len(value)):
		yield value[:i] + value[i + 1:]
		yield (value[:i] + value[i + 1:])

		# for i in range(len(value)):
		# for shrunk_value in self.arbitrary.shrink(value[i]):
		@@ -348,26 +348,45 @@ class ListA(Arbitrary[List[T]]):
		return str([self.arbitrary.to_str(val) for val in value])


		class ChoiceA(Arbitrary[Tuple[int, T]]):
		"""
		ChoiceA provides tuple of (chosen_index, value) where chosen_index
		is the index of the arbitrary used, value is a value provided to said
		arbitrary, it shrinks to (chosen_index, shrunk_value)
		"""
		class _ChoiceA(Arbitrary[Tuple[int, Generatable]]):

		def __init__(self, choices: Sequence[Tuple[int, Arbitrary[T]]]):
		super().__init__(choices_gen([(v, a.generator) for v, a in choices]))
		self.choices = choices
		def __init__(self, generator: Callable[[], Tuple[int, Generatable]],
		arbitraries: List[Arbitrary[Generatable]]):
		super().__init__(generator)
		self.arbitraries = arbitraries

		def shrink(self, value: Tuple[int, T]) \
		-> Generator[Tuple[int, T], None, None]:
		choice_index, choice_val = value

		for v in self.choices[choice_index][1].shrink(choice_val):
		for v in self.arbitraries[choice_index].shrink(choice_val):
		yield (choice_index, v)

		def to_str(self, value: Tuple[int, T]) -> str:
		return self.choices[value[0]][1].to_str(value[1])
		return self.arbitraries[value[0]].to_str(value[1])


		class ChoiceA(_ChoiceA):
		"""
		ChoiceA provides tuple of (chosen_index, value) where chosen_index
		is the index of the arbitrary used, value is a value provided to said
		arbitrary, it shrinks to (chosen_index, shrunk_value)
		"""

		def __init__(self, choices: Sequence[Tuple[int, Arbitrary[Generatable]]]):
		super().__init__(choices_gen([(v, a.generator) for v, a in choices]),
		[a for _, a in choices])


		class ChoiceEQA(Arbitrary[Tuple[int, Generatable]]):
		"""
		ChoiceEQA is the same as ChoiceA, but each arbitrary has equal chance of
		being chosen.
		"""

		def __init__(self, arbitraries: Sequence[Arbitrary[T]]):
		super().__init__(eqchoices_gen([a.generator] for a in arbitraries),
		arbitraries)


		class NumberA(Arbitrary[int]):
		@@ -418,7 +437,7 @@ class Test(Generic[T]):
		def __init__(self, name: str, arbitrary: Arbitrary[T],
		value: Optional[T] = None) -> None:
		self.name = name
		self.value = value if value else arbitrary.get()
		self.value = value if value is not None else arbitrary.get()
		self.arbitrary = arbitrary

		def shrink(self) -> Generator['Test[T]', None, None]:
		@@ -450,7 +469,7 @@ class Tester:
		assert False, "run_test not implemented"

		def log(self, priority: int, args, *kwargs) -> None:
		if priority < self.verbosity:
		if priority <= self.verbosity:
		print(args, *kwargs)

		def find_smallest(self, test: Test[Generatable]) -> Optional[ShrunkTest]:
		@@ -458,7 +477,6 @@ class Tester:
		if(self.run_test(test)):
		return None

		self.log(1, "Failed")

		shrinks = 0
		smallest = test
		@@ -466,17 +484,24 @@ class Tester:

		while testing:
		testing = False
		if(shrinks % 10 == 0):
		self.log(2, f"Shrinks {shrinks}")
		if shrinks % 10 == 0:
		self.log(4, f"Shrinks {shrinks}")

		self.log(7, f"Current shrink ({shrinks}):", smallest.to_str(),
		sep="\n")

		self.log(6, smallest.to_str())
		for sub_test in smallest.shrink():
		if not self.run_test(sub_test):
		self.log(6, "Running test")
		self.log(7, "the test being run:", sub_test.to_str(), sep="\n")
		result = self.run_test(sub_test)
		self.log(6, "Test {'passed' if result else 'failed'}")
		if not result:
		testing = True
		smallest = sub_test
		shrinks += 1
		break

		self.log(2, f"Test shrunk using {shrinks} shrinks")
		self.log(3, smallest.to_str())
		return (smallest, shrinks)

		def run(self) -> List[ShrunkTest]:
		@@ -489,9 +514,11 @@ class Tester:
		for test in self.tests:
		self.log(1, f"Test start: {test.name}")
		shrunk_version = self.find_smallest(test)
		if(shrunk_version):
		if shrunk_version:
		self.log(2, "Failed, shrinking")
		failed_tests.append(shrunk_version)

		else:
		self.log(2, "Passed")
		return failed_tests

		def save_tests(self) -> None:
		@@ -513,7 +540,7 @@ class Tester:


		##############################################################################
		# REFERENCE IMPLEMENTAION TESTING
		# REFERENCE IMPLEMENTATION TESTING
		##############################################################################

		def str_id(x: str) -> str:
		@@ -576,33 +603,42 @@ class RefImplTester(Tester):
		super().__init__(verbosity)
		self.program_tested = program_tested
		self.program_referenced = program_referenced
		self.flags: List[str] = []
		self.args: List[str] = []
		self.test_args: List[str] = []
		self.ref_args: List[str] = []
		self.output_tests: List[ProcessOutputTest] = []

		def set_flags(self, flags: List[str]) -> None:
		self.flags = flags
		def set_args(self, args: List[str]) -> None:
		self.args = args

		def set_test_args(self, test_args: List[str]) -> None:
		self.test_args = test_args

		def set_ref_args(self, ref_args: List[str]) -> None:
		self.ref_args = ref_args

		def run_test(self, test: Test[Generatable]) -> bool:
		self.log(5, "run_test")
		test_string = test.to_str()
		self.log(6, test_string)

		test_program = subprocess.run(
		[self.program_tested] + self.flags,
		[self.program_tested] + self.args + self.test_args,
		capture_output=True,
		input=str_to_bytes(test_string))

		exp_program = subprocess.run(
		[self.program_referenced] + self.flags,
		[self.program_referenced] + self.args + self.ref_args,
		capture_output=True,
		input=str_to_bytes(test_string))

		res = True

		if self.verbosity >= 8:
		self.log(8, "TESTED STDOUT\n" + bytes_to_str(test_program.stdout))
		self.log(8, "REFERENCE STDOUT\n" + bytes_to_str(exp_program.stdout))

		for process_test in self.output_tests:
		res = res and process_test(test_program, exp_program)

		self.log(5, f"test result: {('pass' if res else 'fail')}")
		return res

		def add_output_test(self, test: ProcessOutputTest) -> None: