263 lines
12 KiB
Python
263 lines
12 KiB
Python
"""CLI contract tests: every shipped script, four invocation shapes each.
|
|
|
|
What is asserted, per the repository-wide contract in REBUILD-BRIEF section 4:
|
|
|
|
1. `--help` succeeds and never prints an uncaught traceback.
|
|
2. An unknown flag is a usage error: exit 2.
|
|
3. An invocation with required arguments missing, or with every input pointed
|
|
at a file that does not exist, is a usage error: exit 2.
|
|
4. A script that advertises `--json` emits a parseable JSON document that
|
|
states a `status`.
|
|
5. Every observed exit code is one of 0/1/2/3/4.
|
|
|
|
Two clean outcomes are distinguished, and the distinction is the whole point:
|
|
|
|
* `xfail` means the script is on the tree and does **not** meet the contract.
|
|
The reason carries the measured value. The xfail list this file produces is
|
|
the work order for exit-code unification -- it is not a defect to be fixed
|
|
here, because every one of those scripts has an owner.
|
|
* `skip` means the scenario is not applicable or not reachable on a machine
|
|
with no device, no network and no frida. Those are coverage gaps, and they
|
|
are reported as such in docs/tool-verification/EXTENSION-test-harness.md.
|
|
|
|
Each scenario runs once per script and is cached for the session, so the suite
|
|
spends its time on assertions rather than on re-spawning interpreters.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
|
|
from conftest import all_scripts, run_script
|
|
from contract import (
|
|
CONTRACT_EXIT_CODES, DEVICE_OR_NETWORK_BOUND, JSON_REQUIRED_KEYS,
|
|
JSON_RUNNABLE, execution_refusal, has_option, missing_input_args,
|
|
parse_leading_json,
|
|
)
|
|
|
|
pytestmark = pytest.mark.cli
|
|
|
|
SCRIPTS = all_scripts()
|
|
BAD_FLAG = "--zzz-not-a-real-flag"
|
|
|
|
HELP_TIMEOUT = 60
|
|
SCENARIO_TIMEOUT = 12
|
|
|
|
|
|
def why(name):
|
|
if name in DEVICE_OR_NETWORK_BOUND:
|
|
return " (device/network bound: it may legitimately block)"
|
|
return ""
|
|
|
|
|
|
def require_returned(result, what):
|
|
if result.timed_out:
|
|
pytest.xfail("%s did not return within %ss%s; there is no exit code to check"
|
|
% (what, SCENARIO_TIMEOUT, why(result.argv[1])))
|
|
if result.returncode not in CONTRACT_EXIT_CODES:
|
|
pytest.fail("%s returned %r, outside the 0/1/2/3/4 contract vocabulary"
|
|
% (what, result.returncode))
|
|
|
|
|
|
def expect_exit(result, expected, what):
|
|
"""Pass when the measured code matches; otherwise record it as an xfail."""
|
|
require_returned(result, what)
|
|
if result.returncode != expected:
|
|
pytest.xfail("%s: contract says exit %d, measured %d%s"
|
|
% (what, expected, result.returncode, why(result.argv[1])))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# one cached run per script per scenario
|
|
# --------------------------------------------------------------------------- #
|
|
@pytest.fixture(scope="session")
|
|
def help_results():
|
|
return {n: run_script(n, ["--help"], timeout=HELP_TIMEOUT) for n in SCRIPTS}
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def badflag_results():
|
|
return {n: run_script(n, [BAD_FLAG], timeout=SCENARIO_TIMEOUT) for n in SCRIPTS}
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def empty_results():
|
|
"""Bare invocations, for the scripts that may be run at all.
|
|
|
|
Device-bound scripts are excluded here too: their no-argument path is exactly
|
|
where several of them start doing device work (`usb_net_proxy.py` did not
|
|
return within 20 s when probed by hand), and this suite does not touch a device.
|
|
"""
|
|
return {n: run_script(n, [], timeout=SCENARIO_TIMEOUT) for n in SCRIPTS
|
|
if n not in DEVICE_OR_NETWORK_BOUND}
|
|
|
|
|
|
@pytest.fixture(scope="session")
|
|
def missing_results(help_results):
|
|
"""Argument-complete runs against a missing file, for the scripts that may be run at all.
|
|
|
|
Device-bound scripts are skipped here rather than executed and timed out: on
|
|
the host this harness was written on, `adb devices` reported an online device,
|
|
so running them would have touched real hardware.
|
|
"""
|
|
results = {}
|
|
for name in SCRIPTS:
|
|
if name in DEVICE_OR_NETWORK_BOUND:
|
|
continue
|
|
args = missing_input_args(help_results[name].output)
|
|
if args is None:
|
|
continue
|
|
results[name] = run_script(name, args, timeout=SCENARIO_TIMEOUT)
|
|
return results
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 1. --help
|
|
# --------------------------------------------------------------------------- #
|
|
class TestHelp:
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_help_never_crashes(self, name, help_results):
|
|
"""Hard assertion: an uncaught traceback from --help is a defect, not a drift."""
|
|
result = help_results[name]
|
|
assert not result.timed_out, "--help did not return for %s" % name
|
|
assert not result.has_traceback, result.output[-2000:]
|
|
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_help_prints_something(self, name, help_results):
|
|
assert help_results[name].output.strip(), "--help printed nothing"
|
|
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_help_exits_zero(self, name, help_results):
|
|
expect_exit(help_results[name], 0, "%s --help" % name)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 2. unknown flag
|
|
# --------------------------------------------------------------------------- #
|
|
class TestUnknownFlag:
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_unknown_flag_is_never_success(self, name, badflag_results):
|
|
result = badflag_results[name]
|
|
require_returned(result, "%s %s" % (name, BAD_FLAG))
|
|
assert result.returncode != 0, (
|
|
"an unknown flag was reported as success:\n%s" % result.output[-2000:])
|
|
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_unknown_flag_exits_two(self, name, badflag_results):
|
|
expect_exit(badflag_results[name], 2, "%s %s" % (name, BAD_FLAG))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 3a. empty invocation
|
|
# --------------------------------------------------------------------------- #
|
|
class TestEmptyInvocation:
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_no_arguments_is_a_usage_error(self, name, help_results, empty_results):
|
|
"""Only applies when the script actually requires arguments.
|
|
|
|
A script whose usage block has no positional and no required option may
|
|
legitimately succeed with no arguments; demanding exit 2 from those would
|
|
invent a rule the contract does not state. Device-bound scripts are not run
|
|
at all -- see `contract.execution_refusal`.
|
|
"""
|
|
refusal = execution_refusal(name, "a bare invocation", help_results[name].output)
|
|
if refusal:
|
|
pytest.skip(refusal)
|
|
expect_exit(empty_results[name], 2, "%s with no arguments" % name)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 3b. bad input
|
|
# --------------------------------------------------------------------------- #
|
|
class TestMissingInput:
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_missing_input_file_is_a_usage_error(self, name, help_results, missing_results):
|
|
if name not in missing_results:
|
|
pytest.skip(execution_refusal(name, "a missing input", help_results[name].output)
|
|
or "not executed for %s" % name)
|
|
expect_exit(missing_results[name], 2, "%s with a missing input" % name)
|
|
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_missing_input_is_never_success(self, name, help_results, missing_results):
|
|
if name not in missing_results:
|
|
pytest.skip(execution_refusal(name, "a missing input", help_results[name].output)
|
|
or "not executed for %s" % name)
|
|
result = missing_results[name]
|
|
require_returned(result, "%s with a missing input" % name)
|
|
assert result.returncode != 0, (
|
|
"a missing input was reported as success:\n%s" % result.output[-2000:])
|
|
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_bad_input_reports_cleanly_not_as_an_exception(self, name, help_results,
|
|
missing_results):
|
|
"""A missing file is an expected failure mode; it must not escape as an exception.
|
|
|
|
Several scripts currently let `FileNotFoundError` propagate. Each is
|
|
recorded here as an xfail naming the exception; the fix belongs to that
|
|
script's owner, not to this suite.
|
|
"""
|
|
if name not in missing_results:
|
|
pytest.skip(execution_refusal(name, "a missing input", help_results[name].output)
|
|
or "not executed for %s" % name)
|
|
result = missing_results[name]
|
|
require_returned(result, "%s with a missing input" % name)
|
|
if result.has_traceback:
|
|
tail = [l for l in result.output.splitlines() if l.strip()]
|
|
pytest.xfail("%s reports a missing input by letting an exception escape "
|
|
"instead of exiting 2 cleanly: %s"
|
|
% (name, (tail[-1] if tail else "uncaught exception")[:140]))
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 4. the --json contract
|
|
# --------------------------------------------------------------------------- #
|
|
def json_capable(help_results):
|
|
return sorted(n for n, r in help_results.items() if has_option(r.output, "--json"))
|
|
|
|
|
|
def test_json_capable_scripts_are_discoverable(help_results):
|
|
found = json_capable(help_results)
|
|
assert found, "no script advertises --json; the discovery method is broken"
|
|
|
|
|
|
@pytest.mark.parametrize("name,args", sorted(JSON_RUNNABLE.items()))
|
|
def test_json_output_is_parseable_and_states_a_status(name, args):
|
|
result = run_script(name, args, timeout=120)
|
|
require_returned(result, "%s %s" % (name, " ".join(args)))
|
|
doc = parse_leading_json(result.stdout)
|
|
assert doc is not None, (
|
|
"%s %s printed no parseable JSON document:\n%s"
|
|
% (name, " ".join(args), result.output[-2000:]))
|
|
missing = [k for k in JSON_REQUIRED_KEYS if not isinstance(doc, dict) or k not in doc]
|
|
if missing:
|
|
present = sorted(doc) if isinstance(doc, dict) else type(doc).__name__
|
|
pytest.xfail("%s --json does not carry %s; top-level keys are %s"
|
|
% (name, missing, present))
|
|
|
|
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_json_capable_but_unreachable_scripts_are_named(name, help_results):
|
|
"""Coverage honesty: a --json script is either exercised here or explicitly not."""
|
|
if not has_option(help_results[name].output, "--json"):
|
|
pytest.skip("%s does not advertise --json" % name)
|
|
if name in JSON_RUNNABLE:
|
|
return
|
|
pytest.skip("%s advertises --json, but reaching that path needs a device or a "
|
|
"sample; the gap is listed in EXTENSION-test-harness.md" % name)
|
|
|
|
|
|
# --------------------------------------------------------------------------- #
|
|
# 5. the exit-code vocabulary
|
|
# --------------------------------------------------------------------------- #
|
|
class TestExitCodeVocabulary:
|
|
@pytest.mark.parametrize("name", SCRIPTS)
|
|
def test_observed_codes_stay_inside_the_vocabulary(self, name, help_results,
|
|
badflag_results):
|
|
for label, result in (("--help", help_results[name]),
|
|
(BAD_FLAG, badflag_results[name])):
|
|
if result.returncode is not None:
|
|
assert result.returncode in CONTRACT_EXIT_CODES, (
|
|
"%s %s returned %r" % (name, label, result.returncode))
|
|
assert result.returncode is None or result.returncode >= 0, (
|
|
"a negative code is a crash signal, not an exit status: %s %s"
|
|
% (name, label))
|