From 988a4aac4c00ae0f7512ccb0697a520bb1ec9fc9 Mon Sep 17 00:00:00 2001 From: xenofem Date: Mon, 22 Nov 2021 20:13:45 -0500 Subject: [PATCH 01/12] always display minutes as 2 digits --- impatient | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impatient b/impatient index c5f3bf6..20ac413 100755 --- a/impatient +++ b/impatient @@ -41,7 +41,7 @@ def display_timedelta(d): if d.days != 0: result += '{}d'.format(d.days) result += '{}h'.format(d.seconds // 3600) - result += '{}m'.format((d.seconds % 3600) // 60) + result += '{:02}m'.format((d.seconds % 3600) // 60) return result if args.path: From 6fec8076b4e45264b6c19eec4c6b1a6feba16f65 Mon Sep 17 00:00:00 2001 From: xenofem Date: Mon, 22 Nov 2021 20:40:19 -0500 Subject: [PATCH 02/12] add window size flag, tweak flag documentation --- impatient | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/impatient b/impatient index 20ac413..e8d595e 100755 --- a/impatient +++ b/impatient @@ -6,16 +6,15 @@ import subprocess import sys import time -WINDOW_SIZE = 100 - SUFFIXES = 'KMGT' parser = argparse.ArgumentParser(description='Display progress and time estimates for arbitrary tasks') -parser.add_argument('-f', '--final', type=str, help='Expected final size/value at completion.') +parser.add_argument('-f', '--final', type=str, help='Expected final size/value at completion, optionally with a K/M/G/T suffix') parser.add_argument('-i', '--interval', type=int, default=10, help='Interval in seconds between samples (default 10)') +parser.add_argument('-w', '--window', type=int, default=100, help='Number of samples to keep for the sliding window (default 100)') tracker_types = parser.add_mutually_exclusive_group(required=True) tracker_types.add_argument('-p', '--path', type=str, help='Track total disk usage of a given path') -tracker_types.add_argument('-c', '--command', type=str, help='Track value returned by a shell command; this should return a single number, optionally followed by K/M/G/T') +tracker_types.add_argument('-c', '--command', type=str, help='Track value returned by a shell command; this should return a single number, optionally with a K/M/G/T suffix') args = parser.parse_args() @@ -67,8 +66,8 @@ while True: samples.append(current) if len(samples) < 2: continue - if len(samples) > WINDOW_SIZE: - samples = samples[-WINDOW_SIZE:] + if len(samples) > args.window: + samples = samples[-args.window:] rate = (current - samples[0])/((len(samples)-1)*args.interval) From 918b3bf10b58e607d78a3ff6204edcaaa1dffa43 Mon Sep 17 00:00:00 2001 From: xenofem Date: Mon, 22 Nov 2021 20:40:53 -0500 Subject: [PATCH 03/12] parse float values --- impatient | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/impatient b/impatient index e8d595e..cad3eb1 100755 --- a/impatient +++ b/impatient @@ -22,9 +22,9 @@ def parse_value(v): suffix = v[-1].upper() if suffix in SUFFIXES: exponent = 3*(SUFFIXES.find(suffix)+1) - return int(v[:-1])*(10**exponent) + return float(v[:-1])*(10**exponent) else: - return int(v) + return float(v) def display_value(v): suffix = '' From 105839d8cfabc4e5250f1922868d009a242304e4 Mon Sep 17 00:00:00 2001 From: xenofem Date: Mon, 22 Nov 2021 20:44:15 -0500 Subject: [PATCH 04/12] add README --- README.md | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 README.md diff --git a/README.md b/README.md new file mode 100644 index 0000000..c2f8760 --- /dev/null +++ b/README.md @@ -0,0 +1,54 @@ +# impatient + +A general purpose utility for estimating when a task will complete + +## Overview + +`impatient` is sort of like `pv` for tasks where you can't insert +extra commands into a pipeline, and sort of like `progress` for tasks +that aren't reading/writing a specific file on the local system. If +you give it any shell command that outputs a number (possibly with a +K/M/G/T suffix), it will repeatedly run the command and show you how +fast that number is changing. If you give it the expected final value +of the number, it'll also estimate how much longer you have to wait. +It also has a specific option for tracking the total size of a +file or directory as given by `du`. + +For example, here's `impatient` tracking an in-progress `zfs send` to +a remote machine: + +``` +$ impatient -f "$(zfs list -pH -o used pool/home)" -c 'ssh remote-server zfs list -pH -o used remote-pool/backups/pool/home' -i 60 +109.8G - 2.8M/s - 263.7G total - 41.6% complete - 15h03m remaining - ETA 2021-11-23 11:34 +``` + +Note that `-f` is given the value output by `zfs list`; we could +give it a number directly, this just simplifies things in this +particular scenario. The `-p` flag for `zfs list` isn't strictly +necessary, `impatient` can parse values like `50.7G`, but +using the exact byte value provides more precision. + +`-c`, on the other hand, is given a quoted string containing a +command, which will output the current progress value each time it's +run. The command given to `-c` can be an arbitrary shell one-liner, +use responsibly. + +## Usage + +``` +usage: impatient [-h] [-f FINAL] [-i INTERVAL] [-w WINDOW] (-p PATH | -c COMMAND) + +Display progress and time estimates for arbitrary tasks + +optional arguments: + -h, --help show this help message and exit + -f FINAL, --final FINAL + Expected final size/value at completion, optionally with a K/M/G/T suffix + -i INTERVAL, --interval INTERVAL + Interval in seconds between samples (default 10) + -w WINDOW, --window WINDOW + Number of samples to keep for the sliding window (default 100) + -p PATH, --path PATH Track total disk usage of a given path + -c COMMAND, --command COMMAND + Track value returned by a shell command; this should return a single number, optionally with a K/M/G/T suffix +``` From 93e17e14b8a732d398a27ec005dd0289f74170a0 Mon Sep 17 00:00:00 2001 From: xenofem Date: Tue, 23 Nov 2021 01:39:11 -0500 Subject: [PATCH 05/12] refactor around deltas, add configurable decay coefficient and smarter halting --- impatient | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/impatient b/impatient index cad3eb1..8e18462 100755 --- a/impatient +++ b/impatient @@ -12,11 +12,21 @@ parser = argparse.ArgumentParser(description='Display progress and time estimate parser.add_argument('-f', '--final', type=str, help='Expected final size/value at completion, optionally with a K/M/G/T suffix') parser.add_argument('-i', '--interval', type=int, default=10, help='Interval in seconds between samples (default 10)') parser.add_argument('-w', '--window', type=int, default=100, help='Number of samples to keep for the sliding window (default 100)') +parser.add_argument('-d', '--decay', type=float, default=1, help='Decay coefficient for older samples (default 1). Must be between 0 and 1 inclusive. The lower this is, the more responsive/swingy the estimate will be.') tracker_types = parser.add_mutually_exclusive_group(required=True) tracker_types.add_argument('-p', '--path', type=str, help='Track total disk usage of a given path') tracker_types.add_argument('-c', '--command', type=str, help='Track value returned by a shell command; this should return a single number, optionally with a K/M/G/T suffix') args = parser.parse_args() +if args.interval < 1: + print('interval must be at least 1', file=sys.stderr) + exit(1) +if args.window < 1: + print('window size must be at least 1', file=sys.stderr) + exit(1) +if args.decay < 0 or args.decay > 1: + print('decay coefficient must be between 0 and 1', file=sys.stderr) + exit(1) def parse_value(v): suffix = v[-1].upper() @@ -57,19 +67,26 @@ if args.final: else: final = None -samples = [current_val()] +deltas = [] +current = current_val() while True: time.sleep(args.interval) - current = current_val() + new = current_val() - samples.append(current) - if len(samples) < 2: - continue - if len(samples) > args.window: - samples = samples[-args.window:] + deltas.append(new - current) + current = new + if len(deltas) > args.window: + deltas = deltas[-args.window:] - rate = (current - samples[0])/((len(samples)-1)*args.interval) + total = 0 + divisor = 0 + coeff = 1 + for d in deltas[::-1]: + total += coeff*d + divisor += coeff + coeff *= args.decay + rate = total/(divisor*args.interval) print('\033[2K\r', end='') print('{} - {}/s'.format(display_value(current), display_value(rate)), end='') @@ -86,6 +103,6 @@ while True: ), end='') sys.stdout.flush() - if final and current >= final: + if final and current >= final and all(d == 0 for d in deltas[-10:]): print() break From 4cf9fb137b8a316fe7d82027f66d3bc0b793664a Mon Sep 17 00:00:00 2001 From: xenofem Date: Tue, 23 Nov 2021 01:41:33 -0500 Subject: [PATCH 06/12] update README docs --- README.md | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index c2f8760..fc511dc 100644 --- a/README.md +++ b/README.md @@ -36,19 +36,28 @@ use responsibly. ## Usage ``` -usage: impatient [-h] [-f FINAL] [-i INTERVAL] [-w WINDOW] (-p PATH | -c COMMAND) +usage: impatient [-h] [-f FINAL] [-i INTERVAL] [-w WINDOW] [-d DECAY] + (-p PATH | -c COMMAND) Display progress and time estimates for arbitrary tasks optional arguments: -h, --help show this help message and exit -f FINAL, --final FINAL - Expected final size/value at completion, optionally with a K/M/G/T suffix + Expected final size/value at completion, optionally + with a K/M/G/T suffix -i INTERVAL, --interval INTERVAL Interval in seconds between samples (default 10) -w WINDOW, --window WINDOW - Number of samples to keep for the sliding window (default 100) + Number of samples to keep for the sliding window + (default 100) + -d DECAY, --decay DECAY + Decay coefficient for older samples (default 1). Must + be between 0 and 1 inclusive. The lower this is, the + more responsive/swingy the estimate will be. -p PATH, --path PATH Track total disk usage of a given path -c COMMAND, --command COMMAND - Track value returned by a shell command; this should return a single number, optionally with a K/M/G/T suffix + Track value returned by a shell command; this should + return a single number, optionally with a K/M/G/T + suffix ``` From ad995b0845ef806e742bbaa5ca5e23391fb858ce Mon Sep 17 00:00:00 2001 From: xenofem Date: Tue, 23 Nov 2021 12:46:16 -0500 Subject: [PATCH 07/12] better handling of stalled/overfilled tasks --- impatient | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/impatient b/impatient index 8e18462..278a2a2 100755 --- a/impatient +++ b/impatient @@ -2,6 +2,7 @@ import argparse import datetime +import math import subprocess import sys import time @@ -94,8 +95,13 @@ while True: fraction = current / final value_remaining = final - current print(' - {} total - {:.1f}% complete'.format(display_value(final), 100*fraction), end='') - if rate > 0: - time_remaining = datetime.timedelta(seconds=(value_remaining / rate)) + seconds_remaining = (value_remaining / rate) if rate > 0 else math.inf + if seconds_remaining < 0: + print(' - unknown time remaining', end='') + elif seconds_remaining > 864000: + print(' - stalled', end='') + else: + time_remaining = datetime.timedelta(seconds=seconds_remaining) eta = datetime.datetime.now() + time_remaining print(' - {} remaining - ETA {}'.format( display_timedelta(time_remaining), From f1ac7516b04385e8b5521cba15eaaa31849f6829 Mon Sep 17 00:00:00 2001 From: xenofem Date: Tue, 8 Feb 2022 02:16:11 -0500 Subject: [PATCH 08/12] add customizable thresholds for termination --- impatient | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/impatient b/impatient index 278a2a2..2fa9866 100755 --- a/impatient +++ b/impatient @@ -14,6 +14,8 @@ parser.add_argument('-f', '--final', type=str, help='Expected final size/value a parser.add_argument('-i', '--interval', type=int, default=10, help='Interval in seconds between samples (default 10)') parser.add_argument('-w', '--window', type=int, default=100, help='Number of samples to keep for the sliding window (default 100)') parser.add_argument('-d', '--decay', type=float, default=1, help='Decay coefficient for older samples (default 1). Must be between 0 and 1 inclusive. The lower this is, the more responsive/swingy the estimate will be.') +parser.add_argument('-V', '--termination-value-threshold', type=float, default=0.95, metavar='FRACTION', help='Fraction of the expected final value that must be reached in order to terminate (default 0.95). Reaching this threshold is necessary but not sufficient, see also -I') +parser.add_argument('-I', '--termination-inactivity-threshold', type=int, default=10, metavar='COUNT', help='Number of consecutive unchanged samples that must be observed in order to terminate (default 10). Reaching this threshold is necessary but not sufficient, see also -V') tracker_types = parser.add_mutually_exclusive_group(required=True) tracker_types.add_argument('-p', '--path', type=str, help='Track total disk usage of a given path') tracker_types.add_argument('-c', '--command', type=str, help='Track value returned by a shell command; this should return a single number, optionally with a K/M/G/T suffix') @@ -28,6 +30,12 @@ if args.window < 1: if args.decay < 0 or args.decay > 1: print('decay coefficient must be between 0 and 1', file=sys.stderr) exit(1) +if args.termination_value_threshold < 0 or args.termination_value_threshold > 1: + print('termination value threshold must be between 0 and 1', file=sys.stderr) + exit(1) +if args.termination_inactivity_threshold < 0: + print('termination inactivity threshold must be nonnegative', file=sys.stderr) + exit(1) def parse_value(v): suffix = v[-1].upper() @@ -65,6 +73,9 @@ else: if args.final: final = parse_value(args.final) + if final <= 0: + print('final value must be positive', file=sys.stderr) + exit(1) else: final = None @@ -109,6 +120,6 @@ while True: ), end='') sys.stdout.flush() - if final and current >= final and all(d == 0 for d in deltas[-10:]): + if final and current >= args.termination_value_threshold*final and all(d == 0 for d in deltas[-args.termination_inactivity_threshold:]): print() break From a8a20099235bcd0d675a3396a585c7d9a6b52fb2 Mon Sep 17 00:00:00 2001 From: xenofem Date: Tue, 8 Feb 2022 02:34:15 -0500 Subject: [PATCH 09/12] add option to log time series to a csv file --- impatient | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/impatient b/impatient index 2fa9866..7fa8ff7 100755 --- a/impatient +++ b/impatient @@ -16,6 +16,7 @@ parser.add_argument('-w', '--window', type=int, default=100, help='Number of sam parser.add_argument('-d', '--decay', type=float, default=1, help='Decay coefficient for older samples (default 1). Must be between 0 and 1 inclusive. The lower this is, the more responsive/swingy the estimate will be.') parser.add_argument('-V', '--termination-value-threshold', type=float, default=0.95, metavar='FRACTION', help='Fraction of the expected final value that must be reached in order to terminate (default 0.95). Reaching this threshold is necessary but not sufficient, see also -I') parser.add_argument('-I', '--termination-inactivity-threshold', type=int, default=10, metavar='COUNT', help='Number of consecutive unchanged samples that must be observed in order to terminate (default 10). Reaching this threshold is necessary but not sufficient, see also -V') +parser.add_argument('-l', '--log-file', type=str, metavar='PATH', help='File to log the time series to. Will be saved as a csv, with columns for timestamp and for value. Will append data if the file already exists.') tracker_types = parser.add_mutually_exclusive_group(required=True) tracker_types.add_argument('-p', '--path', type=str, help='Track total disk usage of a given path') tracker_types.add_argument('-c', '--command', type=str, help='Track value returned by a shell command; this should return a single number, optionally with a K/M/G/T suffix') @@ -63,14 +64,25 @@ def display_timedelta(d): return result if args.path: - def current_val(): + def current_val_helper(): du = subprocess.run(['du', '--bytes', '--summarize', args.path], capture_output=True, text=True).stdout return parse_value(du.split()[0]) else: - def current_val(): + def current_val_helper(): result = subprocess.run(args.command, shell=True, capture_output=True, text=True).stdout return parse_value(result.strip()) +if args.log_file: + log_file = open(args.log_file, mode='a', buffering=1) +else: + log_file = None + +def current_val(): + result = current_val_helper() + if log_file: + print('{},{}'.format(datetime.datetime.now(datetime.timezone.utc), result), file=log_file) + return result + if args.final: final = parse_value(args.final) if final <= 0: From 33957399323d54112662944315cdf1e948926539 Mon Sep 17 00:00:00 2001 From: xenofem Date: Tue, 8 Feb 2022 11:59:36 -0500 Subject: [PATCH 10/12] better date formatting in log file --- impatient | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/impatient b/impatient index 7fa8ff7..b285cf7 100755 --- a/impatient +++ b/impatient @@ -80,7 +80,7 @@ else: def current_val(): result = current_val_helper() if log_file: - print('{},{}'.format(datetime.datetime.now(datetime.timezone.utc), result), file=log_file) + print('{},{}'.format(datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S"), result), file=log_file) return result if args.final: From 710fc9fe81ca259ad058c56b292049da3a097215 Mon Sep 17 00:00:00 2001 From: xenofem Date: Thu, 24 Feb 2022 00:34:08 -0500 Subject: [PATCH 11/12] update README with added options --- README.md | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index fc511dc..b59ba65 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ use responsibly. ``` usage: impatient [-h] [-f FINAL] [-i INTERVAL] [-w WINDOW] [-d DECAY] - (-p PATH | -c COMMAND) + [-V FRACTION] [-I COUNT] [-l PATH] (-p PATH | -c COMMAND) Display progress and time estimates for arbitrary tasks @@ -55,6 +55,20 @@ optional arguments: Decay coefficient for older samples (default 1). Must be between 0 and 1 inclusive. The lower this is, the more responsive/swingy the estimate will be. + -V FRACTION, --termination-value-threshold FRACTION + Fraction of the expected final value that must be + reached in order to terminate (default 0.95). Reaching + this threshold is necessary but not sufficient, see + also -I + -I COUNT, --termination-inactivity-threshold COUNT + Number of consecutive unchanged samples that must be + observed in order to terminate (default 10). Reaching + this threshold is necessary but not sufficient, see + also -V + -l PATH, --log-file PATH + File to log the time series to. Will be saved as a + csv, with columns for timestamp and for value. Will + append data if the file already exists. -p PATH, --path PATH Track total disk usage of a given path -c COMMAND, --command COMMAND Track value returned by a shell command; this should From 595c8026bf6587792e579b3a802eb244ddb760c1 Mon Sep 17 00:00:00 2001 From: xenofem Date: Thu, 24 Feb 2022 00:38:04 -0500 Subject: [PATCH 12/12] make sure we have enough samples before checking termination inactivity threshold --- impatient | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/impatient b/impatient index b285cf7..0240e7e 100755 --- a/impatient +++ b/impatient @@ -132,6 +132,11 @@ while True: ), end='') sys.stdout.flush() - if final and current >= args.termination_value_threshold*final and all(d == 0 for d in deltas[-args.termination_inactivity_threshold:]): + if ( + final + and current >= args.termination_value_threshold*final + and len(deltas) >= args.termination_inactivity_threshold + and all(d == 0 for d in deltas[-args.termination_inactivity_threshold:]) + ): print() break