reverse-skills-max: initial public release (restored, redacted, hardened)
This commit is contained in:
baradok609hecl1pq6qi
@@ -0,0 +1,160 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Anti-crawl type classifier.
|
||||
|
||||
Classifies the target website's anti-crawl mechanism into one of three types:
|
||||
1. Signature-type: environment IS the signature (e.g., RS/Akamai, 412 challenges)
|
||||
2. Behavioral-type: parameter signing + interceptors (e.g., TikTok a_bogus)
|
||||
3. Pure obfuscation: just hard to read, no environment detection
|
||||
|
||||
Usage:
|
||||
python3 scripts/classify_anticrawl.py --probe artifacts/probe_dump.json --analysis analysis_result.json --output artifacts/anticrawl_type.json
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from common import dump_json, flatten_events, load_json
|
||||
|
||||
|
||||
def classify(events: list[dict], analysis: dict) -> dict:
|
||||
"""Classify anti-crawl type based on probe events and analysis."""
|
||||
result = {
|
||||
"type": "unknown",
|
||||
"confidence": "low",
|
||||
"evidence": [],
|
||||
"recommended_strategy": "runtime_hook",
|
||||
"capabilities": [],
|
||||
"event_ids": []
|
||||
}
|
||||
|
||||
# Adversarial runtime signals. These are evidence of observed
|
||||
# behavior, not proof that a particular vendor or challenge was identified.
|
||||
# The probe itself calls Function#toString, descriptor APIs and property
|
||||
# getters while installing hooks. Those self-generated events must not be
|
||||
# promoted to a target anti-debug verdict.
|
||||
self_generated = {
|
||||
"patch.install", "probe.ready", "probe.restored",
|
||||
"environment.watch_installed", "patch.reconciled",
|
||||
}
|
||||
adversarial = []
|
||||
explicit_adversarial = []
|
||||
observation_only = []
|
||||
for event in events:
|
||||
etype = str(event.get("type", ""))
|
||||
path = str(event.get("path", ""))
|
||||
if etype in self_generated or etype.startswith("integrity."):
|
||||
continue
|
||||
if etype.startswith("patch.lost"):
|
||||
adversarial.append(("hook_integrity_check", event))
|
||||
explicit_adversarial.append(("hook_integrity_check", event))
|
||||
if etype.startswith("intervention."):
|
||||
adversarial.append(("dynamic_code", event))
|
||||
explicit_adversarial.append(("dynamic_code", event))
|
||||
elif etype.startswith("dynamic."):
|
||||
# Dynamic-code hooks are useful observations, but a call to the
|
||||
# wrapper alone is not evidence that the target is fighting us.
|
||||
observation_only.append(("dynamic_code_observed", event))
|
||||
if etype.startswith("environment.property_read"):
|
||||
adversarial.append(("environment_property", event))
|
||||
if etype.startswith("realm.") or etype.startswith("loader."):
|
||||
adversarial.append(("dynamic_realm_or_loader", event))
|
||||
if "webdriver" in path or "canvas" in path or "webgl" in path:
|
||||
adversarial.append(("fingerprint_property", event))
|
||||
|
||||
# Check for signature-type indicators
|
||||
signature_indicators = []
|
||||
for event in events:
|
||||
etype = event.get("type", "")
|
||||
# 412 challenge patterns
|
||||
if "412" in str(event.get("url", "")):
|
||||
signature_indicators.append("412_challenge")
|
||||
# Environment fingerprinting
|
||||
if "navigator" in str(event.get("function", "")).lower():
|
||||
signature_indicators.append("navigator_access")
|
||||
if "canvas" in str(event.get("function", "")).lower():
|
||||
signature_indicators.append("canvas_fingerprint")
|
||||
if "webgl" in str(event.get("function", "")).lower():
|
||||
signature_indicators.append("webgl_fingerprint")
|
||||
|
||||
# Check for behavioral-type indicators
|
||||
behavioral_indicators = []
|
||||
for event in events:
|
||||
etype = event.get("type", "")
|
||||
# XHR/fetch with signing
|
||||
if etype.startswith("network."):
|
||||
body = str(event.get("input_preview", ""))
|
||||
if any(kw in body.lower() for kw in ["x-bogus", "a_bogus", "wmsdk", "sign", "signature"]):
|
||||
behavioral_indicators.append("signing_parameter")
|
||||
# Crypto library usage
|
||||
if etype.startswith("crypto."):
|
||||
behavioral_indicators.append("crypto_operation")
|
||||
|
||||
# Check for obfuscation indicators
|
||||
obfuscation_indicators = []
|
||||
# Check if there are many nested function calls
|
||||
if len(events) > 100:
|
||||
obfuscation_indicators.append("high_event_count")
|
||||
# Check for eval/Function usage
|
||||
for event in events:
|
||||
if "eval" in str(event.get("function", "")).lower():
|
||||
obfuscation_indicators.append("eval_usage")
|
||||
|
||||
adversarial_names = sorted({name for name, _ in adversarial})
|
||||
result["capabilities"] = adversarial_names
|
||||
result["event_ids"] = [str(event.get("event_id")) for _, event in adversarial[:100] if event.get("event_id")]
|
||||
result["observation_only"] = sorted({name for name, _ in observation_only})
|
||||
result["probe_self_events_excluded"] = True
|
||||
|
||||
# Determine type
|
||||
if signature_indicators:
|
||||
result["type"] = "signature"
|
||||
result["confidence"] = "high" if len(signature_indicators) >= 2 else "medium"
|
||||
result["evidence"] = signature_indicators
|
||||
result["recommended_strategy"] = "environment_masquerade"
|
||||
elif behavioral_indicators:
|
||||
result["type"] = "behavioral"
|
||||
result["confidence"] = "high" if len(behavioral_indicators) >= 2 else "medium"
|
||||
result["evidence"] = behavioral_indicators
|
||||
result["recommended_strategy"] = "algorithm_tracking"
|
||||
elif obfuscation_indicators:
|
||||
result["type"] = "obfuscation"
|
||||
result["confidence"] = "medium"
|
||||
result["evidence"] = obfuscation_indicators
|
||||
result["recommended_strategy"] = "runtime_hook"
|
||||
else:
|
||||
result["type"] = "simple"
|
||||
result["confidence"] = "medium"
|
||||
result["evidence"] = ["no_complex_indicators"]
|
||||
result["recommended_strategy"] = "runtime_hook"
|
||||
|
||||
# Prefer the more specific capability when the adversarial probe produced
|
||||
# stronger evidence than the legacy keyword classifier.
|
||||
if explicit_adversarial:
|
||||
result["type"] = "adversarial_runtime"
|
||||
result["confidence"] = "high" if len(adversarial_names) >= 2 else "medium"
|
||||
result["evidence"] = sorted(set(result["evidence"] + adversarial_names))[:50]
|
||||
result["recommended_strategy"] = (
|
||||
"source_tap" if "dynamic_code" in adversarial_names else "minimal_intervention"
|
||||
)
|
||||
|
||||
return result
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description="Anti-crawl type classifier.")
|
||||
parser.add_argument("--probe", required=True, help="Path to probe_dump.json.")
|
||||
parser.add_argument("--analysis", help="Path to analysis_result.json.")
|
||||
parser.add_argument("--output", required=True, help="Path to output JSON.")
|
||||
args = parser.parse_args()
|
||||
|
||||
probe = load_json(args.probe, {})
|
||||
analysis = load_json(args.analysis, {}) if args.analysis else {}
|
||||
events = flatten_events(probe)
|
||||
|
||||
result = classify(events, analysis)
|
||||
dump_json(args.output, result)
|
||||
print(f"[OK] anti-crawl type: {result['type']} (confidence: {result['confidence']})")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user