Adds most of the toolsHEADmaster

1 files changed, 374 insertions, 0 deletions

diff --git a/v_windows/v/cmd/tools/repeat.v b/v_windows/v/cmd/tools/repeat.v
new file mode 100644
index 0000000..e69cfdf
--- /dev/null
+++ b/v_windows/v/cmd/tools/repeat.v
@@ -0,0 +1,374 @@
+module main
+
+import os
+import flag
+import time
+import term
+import math
+import scripting
+
+struct CmdResult {
+mut:
+	runs    int
+	cmd     string
+	icmd    int
+	outputs []string
+	oms     map[string][]int
+	summary map[string]Aints
+	timings []int
+	atiming Aints
+}
+
+struct Context {
+mut:
+	count                   int
+	series                  int
+	warmup                  int
+	show_help               bool
+	show_output             bool
+	use_newline             bool // use \n instead of \r, so the last line is not overwritten
+	fail_on_regress_percent int
+	fail_on_maxtime         int // in ms
+	verbose                 bool
+	commands                []string
+	results                 []CmdResult
+	cmd_template            string // {T} will be substituted with the current command
+	cmd_params              map[string][]string
+	cline                   string // a terminal clearing line
+	cgoback                 string
+	nmins                   int // number of minimums to discard
+	nmaxs                   int // number of maximums to discard
+}
+
+[unsafe]
+fn (mut result CmdResult) free() {
+	unsafe {
+		result.cmd.free()
+		result.outputs.free()
+		result.oms.free()
+		result.summary.free()
+		result.timings.free()
+		result.atiming.free()
+	}
+}
+
+[unsafe]
+fn (mut context Context) free() {
+	unsafe {
+		context.commands.free()
+		context.results.free()
+		context.cmd_template.free()
+		context.cmd_params.free()
+		context.cline.free()
+		context.cgoback.free()
+	}
+}
+
+struct Aints {
+	values []int
+mut:
+	imin    int
+	imax    int
+	average f64
+	stddev  f64
+	nmins   int // number of discarded fastest results
+	nmaxs   int // number of discarded slowest results
+}
+
+[unsafe]
+fn (mut a Aints) free() {
+	unsafe { a.values.free() }
+}
+
+fn new_aints(ovals []int, extreme_mins int, extreme_maxs int) Aints {
+	mut res := Aints{
+		values: ovals // remember the original values
+		nmins: extreme_mins
+		nmaxs: extreme_maxs
+	}
+	mut sum := i64(0)
+	mut imin := math.max_i32
+	mut imax := -math.max_i32
+	// discard the extremes:
+	mut vals := []int{}
+	for x in ovals {
+		vals << x
+	}
+	vals.sort()
+	if vals.len > extreme_mins + extreme_maxs {
+		vals = vals[extreme_mins..vals.len - extreme_maxs].clone()
+	} else {
+		vals = []
+	}
+	// statistical processing of the remaining values:
+	for i in vals {
+		sum += i
+		if i < imin {
+			imin = i
+		}
+		if i > imax {
+			imax = i
+		}
+	}
+	res.imin = imin
+	res.imax = imax
+	if vals.len > 0 {
+		res.average = sum / f64(vals.len)
+	}
+	//
+	mut devsum := f64(0.0)
+	for i in vals {
+		x := f64(i) - res.average
+		devsum += (x * x)
+	}
+	res.stddev = math.sqrt(devsum / f64(vals.len))
+	// eprintln('\novals: $ovals\n vals: $vals\n vals.len: $vals.len |  res.imin: $res.imin | res.imax: $res.imax | res.average: $res.average | res.stddev: $res.stddev')
+	return res
+}
+
+fn bold(s string) string {
+	return term.colorize(term.bold, s)
+}
+
+fn (a Aints) str() string {
+	return bold('${a.average:6.2f}') +
+		'ms ± σ: ${a.stddev:4.1f}ms, min: ${a.imin:4}ms, max: ${a.imax:4}ms, runs:${a.values.len:3}, nmins:${a.nmins:2}, nmaxs:${a.nmaxs:2}'
+}
+
+const (
+	max_fail_percent             = 100 * 1000
+	max_time                     = 60 * 1000 // ms
+	performance_regression_label = 'Performance regression detected, failing since '
+)
+
+fn main() {
+	mut context := Context{}
+	context.parse_options()
+	context.run()
+	context.show_diff_summary()
+}
+
+fn (mut context Context) parse_options() {
+	mut fp := flag.new_flag_parser(os.args)
+	fp.application(os.file_name(os.executable()))
+	fp.version('0.0.1')
+	fp.description('Repeat command(s) and collect statistics. NB: you have to quote each command, if it contains spaces.')
+	fp.arguments_description('CMD1 CMD2 ...')
+	fp.skip_executable()
+	fp.limit_free_args_to_at_least(1)
+	context.count = fp.int('count', `c`, 10, 'Repetition count.')
+	context.series = fp.int('series', `s`, 2, 'Series count. `-s 2 -c 4 a b` => aaaabbbbaaaabbbb, while `-s 3 -c 2 a b` => aabbaabbaabb.')
+	context.warmup = fp.int('warmup', `w`, 2, 'Warmup runs. These are done *only at the start*, and are ignored.')
+	context.show_help = fp.bool('help', `h`, false, 'Show this help screen.')
+	context.use_newline = fp.bool('newline', `n`, false, 'Use \\n, do not overwrite the last line. Produces more output, but easier to diagnose.')
+	context.show_output = fp.bool('output', `O`, false, 'Show command stdout/stderr in the progress indicator for each command. NB: slower, for verbose commands.')
+	context.verbose = fp.bool('verbose', `v`, false, 'Be more verbose.')
+	context.fail_on_maxtime = fp.int('max_time', `m`, max_time, 'Fail with exit code 2, when first cmd takes above M milliseconds (regression).')
+	context.fail_on_regress_percent = fp.int('fail_percent', `f`, max_fail_percent, 'Fail with exit code 3, when first cmd is X% slower than the rest (regression).')
+	context.cmd_template = fp.string('template', `t`, '{T}', 'Command template. {T} will be substituted with the current command.')
+	cmd_params := fp.string_multi('parameter', `p`, 'A parameter substitution list. `{p}=val1,val2,val2` means that {p} in the template, will be substituted with each of val1, val2, val3.')
+	context.nmins = fp.int('nmins', `i`, 0, 'Ignore the BOTTOM X results (minimum execution time). Makes the results more robust to performance flukes.')
+	context.nmaxs = fp.int('nmaxs', `a`, 1, 'Ignore the TOP X results (maximum execution time). Makes the results more robust to performance flukes.')
+	for p in cmd_params {
+		parts := p.split(':')
+		if parts.len > 1 {
+			context.cmd_params[parts[0]] = parts[1].split(',')
+		}
+	}
+	if context.show_help {
+		println(fp.usage())
+		exit(0)
+	}
+	if context.verbose {
+		scripting.set_verbose(true)
+	}
+	commands := fp.finalize() or {
+		eprintln('Error: $err')
+		exit(1)
+	}
+	context.commands = context.expand_all_commands(commands)
+	context.results = []CmdResult{len: context.commands.len, cap: 20, init: CmdResult{
+		outputs: []string{cap: 500}
+		timings: []int{cap: 500}
+	}}
+	if context.use_newline {
+		context.cline = '\n'
+		context.cgoback = '\n'
+	} else {
+		context.cline = '\r' + term.h_divider('')
+		context.cgoback = '\r'
+	}
+}
+
+fn (mut context Context) clear_line() {
+	print(context.cline)
+}
+
+fn (mut context Context) expand_all_commands(commands []string) []string {
+	mut all_commands := []string{}
+	for cmd in commands {
+		maincmd := context.cmd_template.replace('{T}', cmd)
+		mut substituted_commands := []string{}
+		substituted_commands << maincmd
+		for paramk, paramlist in context.cmd_params {
+			for paramv in paramlist {
+				mut new_substituted_commands := []string{}
+				for cscmd in substituted_commands {
+					scmd := cscmd.replace(paramk, paramv)
+					new_substituted_commands << scmd
+				}
+				for sc in new_substituted_commands {
+					substituted_commands << sc
+				}
+			}
+		}
+		for sc in substituted_commands {
+			all_commands << sc
+		}
+	}
+	mut unique := map[string]int{}
+	for x in all_commands {
+		if x.contains('{') && x.contains('}') {
+			continue
+		}
+		unique[x] = 1
+	}
+	return unique.keys()
+}
+
+fn (mut context Context) run() {
+	mut run_warmups := 0
+	for si in 1 .. context.series + 1 {
+		for icmd, cmd in context.commands {
+			mut runs := 0
+			mut duration := 0
+			mut sum := 0
+			mut oldres := ''
+			println('Series: ${si:4}/${context.series:-4}, command: $cmd')
+			if context.warmup > 0 && run_warmups < context.commands.len {
+				for i in 1 .. context.warmup + 1 {
+					print('${context.cgoback}warming up run: ${i:4}/${context.warmup:-4} for ${cmd:-50s} took ${duration:6} ms ...')
+					mut sw := time.new_stopwatch()
+					res := os.execute(cmd)
+					if res.exit_code != 0 {
+						continue
+					}
+					duration = int(sw.elapsed().milliseconds())
+				}
+				run_warmups++
+			}
+			context.clear_line()
+			for i in 1 .. (context.count + 1) {
+				avg := f64(sum) / f64(i)
+				print('${context.cgoback}Average: ${avg:9.3f}ms | run: ${i:4}/${context.count:-4} | took ${duration:6} ms')
+				if context.show_output {
+					print(' | result: ${oldres:s}')
+				}
+				mut sw := time.new_stopwatch()
+				res := scripting.exec(cmd) or { continue }
+				duration = int(sw.elapsed().milliseconds())
+				if res.exit_code != 0 {
+					eprintln('${i:10} non 0 exit code for cmd: $cmd')
+					continue
+				}
+				trimed_output := res.output.trim_right('\r\n')
+				trimed_normalized := trimed_output.replace('\r\n', '\n')
+				lines := trimed_normalized.split('\n')
+				for line in lines {
+					context.results[icmd].outputs << line
+				}
+				context.results[icmd].timings << duration
+				sum += duration
+				runs++
+				oldres = res.output.replace('\n', ' ')
+			}
+			context.results[icmd].cmd = cmd
+			context.results[icmd].icmd = icmd
+			context.results[icmd].runs += runs
+			context.results[icmd].atiming = new_aints(context.results[icmd].timings, context.nmins,
+				context.nmaxs)
+			context.clear_line()
+			print(context.cgoback)
+			mut m := map[string][]int{}
+			ioutputs := context.results[icmd].outputs
+			for o in ioutputs {
+				x := o.split(':')
+				if x.len > 1 {
+					k := x[0]
+					v := x[1].trim_left(' ').int()
+					m[k] << v
+				}
+			}
+			mut summary := map[string]Aints{}
+			for k, v in m {
+				// show a temporary summary for the current series/cmd cycle
+				s := new_aints(v, context.nmins, context.nmaxs)
+				println('  $k: $s')
+				summary[k] = s
+			}
+			// merge current raw results to the previous ones
+			old_oms := context.results[icmd].oms.move()
+			mut new_oms := map[string][]int{}
+			for k, v in m {
+				if old_oms[k].len == 0 {
+					new_oms[k] = v
+				} else {
+					new_oms[k] << old_oms[k]
+					new_oms[k] << v
+				}
+			}
+			context.results[icmd].oms = new_oms.move()
+			// println('')
+		}
+	}
+	// create full summaries, taking account of all runs
+	for icmd in 0 .. context.results.len {
+		mut new_full_summary := map[string]Aints{}
+		for k, v in context.results[icmd].oms {
+			new_full_summary[k] = new_aints(v, context.nmins, context.nmaxs)
+		}
+		context.results[icmd].summary = new_full_summary.move()
+	}
+}
+
+fn (mut context Context) show_diff_summary() {
+	context.results.sort_with_compare(fn (a &CmdResult, b &CmdResult) int {
+		if a.atiming.average < b.atiming.average {
+			return -1
+		}
+		if a.atiming.average > b.atiming.average {
+			return 1
+		}
+		return 0
+	})
+	println('Summary (commands are ordered by ascending mean time), after $context.series series of $context.count repetitions:')
+	base := context.results[0].atiming.average
+	mut first_cmd_percentage := f64(100.0)
+	mut first_marker := ''
+	for i, r in context.results {
+		first_marker = ' '
+		cpercent := (r.atiming.average / base) * 100 - 100
+		if r.icmd == 0 {
+			first_marker = bold('>')
+			first_cmd_percentage = cpercent
+		}
+		println(' $first_marker${(i + 1):3} | ${cpercent:5.1f}% slower | ${r.cmd:-57s} | $r.atiming')
+	}
+	$if debugcontext ? {
+		println('context: $context')
+	}
+	if int(base) > context.fail_on_maxtime {
+		print(performance_regression_label)
+		println('average time: ${base:6.1f} ms > $context.fail_on_maxtime ms threshold.')
+		exit(2)
+	}
+	if context.fail_on_regress_percent == max_fail_percent || context.results.len < 2 {
+		return
+	}
+	fail_threshold_max := f64(context.fail_on_regress_percent)
+	if first_cmd_percentage > fail_threshold_max {
+		print(performance_regression_label)
+		println('${first_cmd_percentage:5.1f}% > ${fail_threshold_max:5.1f}% threshold.')
+		exit(3)
+	}
+}