reverse-skills-max: initial public release (restored, redacted, hardened)
This commit is contained in:
baradok609hecl1pq6qi
@@ -0,0 +1,199 @@
|
||||
"""Regression tests for the documented-command gate itself.
|
||||
|
||||
`check_commands.py` is the gate that proves other documentation is runnable, so a blind spot in it
|
||||
is worth more than a blind spot elsewhere: it turns "146 commands checked, 0 drift" into a claim the
|
||||
repository cannot back. One such blind spot is pinned here, exactly as it was found.
|
||||
|
||||
**What was wrong.** `resolve_script()` tried the *document's own directory* first when resolving a
|
||||
relative path. A bare `python analyze_pair.py` written inside `docs/tool-verification/` therefore
|
||||
resolved to `tools/_work/bench/unpack/b3/analyze_pair.py` -- a real file, inside the gitignored
|
||||
workbench -- and scored `ok`. Seven commands in two evidence files used that form while the same
|
||||
files wrote the correct path elsewhere, and a reader copying one from the repository root gets
|
||||
"No such file". The gate said the documentation was clean.
|
||||
|
||||
The tests below drive the resolver and the finding classifier directly rather than through the CLI,
|
||||
so they stay fast and do not depend on the current state of the documentation.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from conftest import REPO_ROOT
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
CHECK_COMMANDS = REPO_ROOT / "check_commands.py"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def gate():
|
||||
"""The gate as a module, so its internals can be driven without a subprocess."""
|
||||
sys.path.insert(0, str(REPO_ROOT))
|
||||
import importlib
|
||||
|
||||
return importlib.import_module("check_commands")
|
||||
|
||||
|
||||
def _workbench_script_name(gate):
|
||||
"""A `.py` basename that exists only under tools/, or skip if the workbench is absent."""
|
||||
names = sorted(gate._workbench_names())
|
||||
if not names:
|
||||
pytest.skip("no workbench present (tools/ is gitignored and may not exist in a checkout)")
|
||||
shipped = {p.name for p in (REPO_ROOT / "skills" / "apk-reverse" / "scripts").glob("*.py")}
|
||||
root = {p.name for p in REPO_ROOT.glob("*.py")}
|
||||
for name in names:
|
||||
if name not in shipped and name not in root:
|
||||
return name
|
||||
pytest.skip("every workbench script name is also shipped; nothing to test with")
|
||||
|
||||
|
||||
def test_a_bare_name_that_only_exists_in_the_workbench_is_drift(gate):
|
||||
"""The exact defect: the name exists *somewhere*, which used to be enough to pass it."""
|
||||
name = _workbench_script_name(gate)
|
||||
doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-extraction-shell-bench.md")
|
||||
path, why = gate.resolve_script(name, doc, include_workbench=False)
|
||||
assert path is None
|
||||
assert why == "unqualified-workbench", why
|
||||
|
||||
text = "```\n$ python %s --flag\n```\n" % name
|
||||
findings = gate.command_findings(doc, text, {}, False)
|
||||
kinds = [f["kind"] for f in findings]
|
||||
assert "unqualified-workbench" in kinds, findings
|
||||
|
||||
|
||||
def test_a_path_qualified_workbench_reference_is_a_skip_not_drift(gate):
|
||||
"""`tools/_work/...` is deliberate in an evidence record: a skip, reported with its reason."""
|
||||
name = _workbench_script_name(gate)
|
||||
doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-extraction-shell-bench.md")
|
||||
_path, why = gate.resolve_script("tools/_work/bench/unpack/b3/%s" % name, doc,
|
||||
include_workbench=False)
|
||||
assert why == "workbench"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def fake_repo(gate, tmp_path, monkeypatch):
|
||||
"""A repository-shaped tree with its own workbench, so resolution can be tested anywhere.
|
||||
|
||||
This exists because the workbench-dependent tests above cannot run on a clean checkout: the
|
||||
`tools/` tree is gitignored, so on CI they skip and the gate's most important property goes
|
||||
unverified exactly where it matters. The tree built here has the layout the resolver assumes
|
||||
(a root, a skill dir, a gitignored `tools/`) and nothing else.
|
||||
"""
|
||||
root = tmp_path / "repo"
|
||||
(root / "skills" / "apk-reverse" / "scripts").mkdir(parents=True)
|
||||
(root / "docs" / "tool-verification").mkdir(parents=True)
|
||||
(root / "tools" / "_work" / "bench").mkdir(parents=True)
|
||||
(root / "README.md").write_text("# root\n", encoding="utf-8")
|
||||
(root / "skills" / "apk-reverse" / "scripts" / "shipped.py").write_text(
|
||||
'"""capability: x\n"""\nimport argparse\np = argparse.ArgumentParser()\n'
|
||||
'p.add_argument("--flag")\n', encoding="utf-8")
|
||||
(root / "tools" / "_work" / "bench" / "solo_in_workbench.py").write_text(
|
||||
"print('x')\n", encoding="utf-8")
|
||||
(root / "docs" / "tool-verification" / "record.md").write_text("# record\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(gate, "ROOT", str(root))
|
||||
monkeypatch.setattr(gate, "SKILL_DIR", str(root / "skills" / "apk-reverse"))
|
||||
monkeypatch.setattr(gate, "SCRIPT_DIR", str(root / "skills" / "apk-reverse" / "scripts"))
|
||||
gate._WORKBENCH_INDEX.clear()
|
||||
yield root
|
||||
gate._WORKBENCH_INDEX.clear()
|
||||
|
||||
|
||||
def test_resolution_order_is_root_then_skill_then_document(gate, fake_repo):
|
||||
"""The corrected order, asserted against a tree that exists on any machine."""
|
||||
doc = str(fake_repo / "docs" / "tool-verification" / "record.md")
|
||||
path, why = gate.resolve_script("README.md", doc, include_workbench=False)
|
||||
assert why == "ok" and path.endswith("README.md")
|
||||
path, why = gate.resolve_script("shipped.py", doc, include_workbench=False)
|
||||
assert why == "ok" and path.endswith("shipped.py")
|
||||
|
||||
|
||||
def test_a_workbench_only_name_is_drift_and_a_qualified_one_is_not(gate, fake_repo):
|
||||
doc = str(fake_repo / "docs" / "tool-verification" / "record.md")
|
||||
path, why = gate.resolve_script("solo_in_workbench.py", doc, include_workbench=False)
|
||||
assert (path, why) == (None, "unqualified-workbench")
|
||||
path, why = gate.resolve_script("tools/_work/bench/solo_in_workbench.py", doc,
|
||||
include_workbench=False)
|
||||
assert (path, why) == (None, "workbench")
|
||||
|
||||
|
||||
def test_a_bare_name_the_document_paths_elsewhere_is_a_skip(gate, fake_repo):
|
||||
"""The fix that made the gate behave the same on CI as on a populated machine.
|
||||
|
||||
`solo_in_workbench.py` appears once bare and once with a path in the same document. The
|
||||
document has already told the reader where it lives, so the bare line is a shorthand -- and
|
||||
that verdict must not depend on the gitignored tree being visible, which is how the first
|
||||
version of this fix passed locally and failed on CI.
|
||||
"""
|
||||
doc = str(fake_repo / "docs" / "tool-verification" / "record.md")
|
||||
text = ("Path: `tools/_work/bench/solo_in_workbench.py` explains it.\n\n"
|
||||
"```\n$ python solo_in_workbench.py\n```\n")
|
||||
findings = gate.command_findings(doc, text, {}, False)
|
||||
kinds = [f["kind"] for f in findings]
|
||||
assert "unqualified-workbench" not in kinds, findings
|
||||
assert "missing-script" not in kinds, findings
|
||||
assert "skipped" in kinds, findings
|
||||
|
||||
|
||||
def test_a_bare_name_after_cd_with_no_resolution_is_skipped_not_drift(gate, fake_repo):
|
||||
"""`cd <somewhere cloned>` then a bare name: unverifiable, and reported as such."""
|
||||
doc = str(fake_repo / "docs" / "tool-verification" / "record.md")
|
||||
text = "```\n$ cd tools/_work/bench/repos/elsewhere\n$ python upstream_tool.py --x\n```\n"
|
||||
findings = gate.command_findings(doc, text, {}, False)
|
||||
kinds = [f["kind"] for f in findings]
|
||||
assert "missing-script" not in kinds, findings
|
||||
assert "after-cd-unresolved" in kinds, findings
|
||||
|
||||
|
||||
def test_the_external_dcc_entry_point_is_never_drift(gate):
|
||||
"""`python dcc.py` after `cd tools/_work/bench/repos/dcc` runs inside the external dcc
|
||||
checkout, so it is a skip on every machine -- including one without the workbench.
|
||||
|
||||
The name is declared in EXTERNAL_SCRIPTS because recognition by filename needs `tools/` to
|
||||
exist: on a clean checkout `dcc.py` degraded to `missing-script` drift and the gate failed away
|
||||
from the machine that wrote the record. Asserted as a *property* rather than as one specific
|
||||
reason, because two mechanisms can legitimately produce it -- the declared entry point and the
|
||||
workbench-after-`cd` rule -- and pinning either one alone makes the test machine-dependent,
|
||||
which is the very defect it is guarding against.
|
||||
"""
|
||||
doc = str(REPO_ROOT / "docs" / "tool-verification" / "EXTENSION-java2c.md")
|
||||
_path, why = gate.resolve_script("dcc.py", doc, include_workbench=False, chdir=True)
|
||||
assert why in ("external", "workbench-after-cd"), why
|
||||
|
||||
text = "```\n$ cd tools/_work/bench/repos/dcc\n$ python dcc.py --no-build\n```\n"
|
||||
findings = gate.command_findings(doc, text, {}, False)
|
||||
kinds = [f["kind"] for f in findings]
|
||||
assert "missing-script" not in kinds, findings
|
||||
assert "unqualified-workbench" not in kinds, findings
|
||||
assert any(k in ("workbench-after-cd", "external") for k in kinds), findings
|
||||
|
||||
|
||||
def test_a_shipped_script_resolves_from_the_repository_root(gate):
|
||||
"""The order that matters: root, then the skill's scripts, then the document's own directory."""
|
||||
doc = str(REPO_ROOT / "docs" / "tool-verification" / "README.md")
|
||||
path, why = gate.resolve_script("check_repo.py", doc, include_workbench=False)
|
||||
assert why == "ok" and path and path.endswith("check_repo.py")
|
||||
path, why = gate.resolve_script("skills/apk-reverse/scripts/scan_leaks.py", doc,
|
||||
include_workbench=False)
|
||||
assert why == "ok" and path and path.endswith("scan_leaks.py")
|
||||
|
||||
|
||||
def test_the_committed_tree_reports_no_drift(gate, tmp_path):
|
||||
"""The gate's own verdict on this repository, run in-process so it cannot drift silently."""
|
||||
docs = gate.collect_docs(None)
|
||||
findings = []
|
||||
cache = {}
|
||||
for doc in docs:
|
||||
try:
|
||||
text = Path(doc).read_text(encoding="utf-8")
|
||||
except OSError:
|
||||
continue
|
||||
findings.extend(gate.command_findings(doc, text, cache, False))
|
||||
drift = [f for f in findings
|
||||
if f["kind"] in ("unknown-flag", "missing-script", "unqualified-workbench")]
|
||||
detail = "\n".join(" %s:%s %s (%s)" % (f["doc"], f["line"], f["command"], f["kind"])
|
||||
for f in drift[:20])
|
||||
assert not drift, "documented commands that cannot run:\n%s" % detail
|
||||
Reference in New Issue
Block a user