KeyHunter parser v10 + 18 scanners + tests (synthetic fixtures only, no keys/db/tokens)
This commit is contained in:
380
agent.py
Normal file
380
agent.py
Normal file
@@ -0,0 +1,380 @@
|
||||
"""KEYHUNTER AGENT — автономный оператор парсер-фермы (Apodex-стиль).
|
||||
|
||||
Отдельный софт: FastAPI :8779 + дашборд + автопилот движков.
|
||||
Движки (gh_eternal, bitbucket, codeberg, clojars, cran, crates, goproxy)
|
||||
запускаются как подпроцессы, статус в памяти, ключи в data/keys.db.
|
||||
Запуск: python agent.py (или python agent.py --port 8779)
|
||||
"""
|
||||
import argparse
|
||||
import os
|
||||
import sqlite3
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import HTMLResponse, PlainTextResponse
|
||||
|
||||
ROOT = Path(__file__).resolve().parent
|
||||
DB = ROOT / "data" / "keys.db"
|
||||
|
||||
# ---------------------------------------------------------------- движки
|
||||
ENGINES = {
|
||||
"gh": {"script": "gh_eternal.py", "log": "gh_eternal.log", "desc": "GitHub dork-hunt 787"},
|
||||
"bitbucket": {"script": "bitbucket_eternal.py", "log": "bitbucket_eternal.log", "desc": "Bitbucket repos"},
|
||||
"codeberg": {"script": "codeberg_eternal.py", "log": "codeberg_eternal.log", "desc": "Codeberg repos"},
|
||||
"clojars": {"script": "clojars_eternal.py", "log": "clojars_eternal.log", "desc": "Clojars (maven)"},
|
||||
"cran": {"script": "cran_eternal.py", "log": "cran_eternal.log", "desc": "CRAN packages"},
|
||||
"crates": {"script": "crates_eternal.py", "log": "crates_eternal.log", "desc": "crates.io"},
|
||||
"goproxy": {"script": "goproxy_eternal.py", "log": "goproxy_eternal.log", "desc": "Go proxy"},
|
||||
}
|
||||
|
||||
_proc: dict = {} # engine_id -> Popen
|
||||
_proc_lock = threading.Lock()
|
||||
AUTOS = {"gh", "crates", "goproxy"} # автопилот: держим живыми
|
||||
|
||||
|
||||
def _spawn(eid: str) -> bool:
|
||||
eng = ENGINES.get(eid)
|
||||
if not eng or not (ROOT / eng["script"]).exists():
|
||||
return False
|
||||
with _proc_lock:
|
||||
p = _proc.get(eid)
|
||||
if p and p.poll() is None:
|
||||
return True
|
||||
# ponytail: CREATE_NEW_PROCESS_GROUP -> корректный terminate на Windows
|
||||
_proc[eid] = subprocess.Popen(
|
||||
[sys.executable, str(ROOT / eng["script"])],
|
||||
cwd=str(ROOT),
|
||||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||||
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0,
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def _stop(eid: str) -> bool:
|
||||
with _proc_lock:
|
||||
p = _proc.pop(eid, None)
|
||||
if not p:
|
||||
return False
|
||||
try:
|
||||
p.terminate()
|
||||
p.wait(timeout=8)
|
||||
except Exception:
|
||||
try: p.kill()
|
||||
except Exception: pass
|
||||
return True
|
||||
|
||||
|
||||
def engines_alive() -> dict:
|
||||
with _proc_lock:
|
||||
items = dict(_proc)
|
||||
return {k: (p.poll() is None) for k, p in items.items()}
|
||||
|
||||
|
||||
def autopilot_loop():
|
||||
"""Раз в минуту: поднимать движки из AUTOS, мёртвые рестартить."""
|
||||
while True:
|
||||
for eid in AUTOS:
|
||||
alive = engines_alive().get(eid, False)
|
||||
if not alive:
|
||||
_spawn(eid)
|
||||
time.sleep(60)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- db
|
||||
def q(sql: str, args=()):
|
||||
con = sqlite3.connect(f"file:{DB}?mode=ro", uri=True, timeout=30)
|
||||
try:
|
||||
con.row_factory = sqlite3.Row
|
||||
return [dict(r) for r in con.execute(sql, args).fetchall()]
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def stats() -> dict:
|
||||
total = q("SELECT COUNT(*) n FROM keys")[0]["n"]
|
||||
by_status = q("SELECT status, COUNT(*) n FROM keys GROUP BY status ORDER BY n DESC")
|
||||
by_prov = q("SELECT prov, COUNT(*) n FROM keys GROUP BY prov ORDER BY n DESC LIMIT 20")
|
||||
today = q(
|
||||
"SELECT COUNT(*) n FROM keys WHERE found LIKE ?",
|
||||
(time.strftime("%Y-%m-%d") + "%",),
|
||||
)[0]["n"]
|
||||
alive = engines_alive()
|
||||
return {
|
||||
"total": total,
|
||||
"today": today,
|
||||
"by_status": by_status,
|
||||
"by_prov": by_prov,
|
||||
"engines": {k: {"alive": alive.get(k, False),
|
||||
"auto": k in AUTOS,
|
||||
"desc": v["desc"]} for k, v in ENGINES.items()},
|
||||
"ts": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||||
}
|
||||
|
||||
|
||||
def tail(path: Path, n=80) -> str:
|
||||
try:
|
||||
with open(path, "rb") as f:
|
||||
f.seek(0, 2)
|
||||
size = f.tell()
|
||||
f.seek(max(0, size - 200_000))
|
||||
chunk = f.read().decode("utf-8", errors="replace")
|
||||
return "\n".join(chunk.splitlines()[-n:])
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- api
|
||||
app = FastAPI(title="KeyHunter Agent")
|
||||
|
||||
|
||||
@app.get("/api/stats")
|
||||
def api_stats():
|
||||
return stats()
|
||||
|
||||
|
||||
@app.get("/api/keys")
|
||||
def api_keys(status: str = "", prov: str = "", limit: int = 50, offset: int = 0):
|
||||
limit = max(1, min(limit, 500))
|
||||
sql, args = "SELECT id, prov, status, plan, price, remaining, found, repo, val FROM keys", []
|
||||
w = []
|
||||
if status:
|
||||
w.append("status=?"); args.append(status)
|
||||
if prov:
|
||||
w.append("prov=?"); args.append(prov)
|
||||
if w:
|
||||
sql += " WHERE " + " AND ".join(w)
|
||||
sql += " ORDER BY id DESC LIMIT ? OFFSET ?"
|
||||
rows = q(sql, (*args, limit, offset))
|
||||
for r in rows: # маскируем ключи в выдаче
|
||||
v = r.get("val") or ""
|
||||
r["val"] = (v[:6] + "…" + v[-4:]) if len(v) > 14 else v
|
||||
return {"rows": rows, "offset": offset, "limit": limit}
|
||||
|
||||
|
||||
@app.get("/api/log")
|
||||
def api_log(engine: str = "gh", n: int = 80):
|
||||
eng = ENGINES.get(engine)
|
||||
path = ROOT / (eng["log"] if eng else "gh_eternal.log")
|
||||
return PlainTextResponse(tail(Path(path), min(n, 400)) or "(лог пуст)")
|
||||
|
||||
|
||||
@app.post("/api/engine/{eid}/start")
|
||||
async def api_start(eid: str):
|
||||
ok = _spawn(eid)
|
||||
return {"ok": ok, "alive": engines_alive().get(eid, False)}
|
||||
|
||||
|
||||
@app.post("/api/engine/{eid}/stop")
|
||||
async def api_stop(eid: str):
|
||||
ok = _stop(eid)
|
||||
return {"ok": ok}
|
||||
|
||||
|
||||
@app.get("/api/export")
|
||||
def api_export(status: str = "NEW", prov: str = ""):
|
||||
sql, args = "SELECT val, prov, status, repo FROM keys", []
|
||||
w = []
|
||||
if status:
|
||||
w.append("status=?"); args.append(status)
|
||||
if prov:
|
||||
w.append("prov=?"); args.append(prov)
|
||||
if w:
|
||||
sql += " WHERE " + " AND ".join(w)
|
||||
body = "\n".join(f"{r['val']}\t{r['prov']}\t{r['repo']}" for r in q(sql, args))
|
||||
return PlainTextResponse(body)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
def health():
|
||||
return {"ok": True, "engines": engines_alive()}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- dashboard
|
||||
_CSS = """
|
||||
:root{--bg:#0b0e14;--panel:#121826;--line:#1e2839;--fg:#c9d4e3;--dim:#5d6b80;--acc:#38bdf8;--ok:#34d399;--warn:#fbbf24;--bad:#f87171}
|
||||
*{margin:0;padding:0;box-sizing:border-box}
|
||||
body{background:var(--bg);color:var(--fg);font:15px/1.55 ui-sans-serif,system-ui,'Segoe UI',Roboto,sans-serif}
|
||||
.wrap{max-width:1080px;margin:0 auto;padding:28px 20px 60px}
|
||||
h1{font-size:26px;letter-spacing:-.02em}
|
||||
h1 b{color:var(--acc)}
|
||||
.sub{color:var(--dim);margin:6px 0 22px}
|
||||
.hero{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin-bottom:24px}
|
||||
.card{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:16px 18px}
|
||||
.card .k{font-size:12px;text-transform:uppercase;letter-spacing:.08em;color:var(--dim)}
|
||||
.card .v{font-size:26px;font-weight:700;margin-top:4px}
|
||||
.card .v.acc{color:var(--acc)}.card .v.ok{color:var(--ok)}
|
||||
h2{font-size:17px;margin:28px 0 12px;display:flex;gap:10px;align-items:baseline}
|
||||
h2 small{color:var(--dim);font-weight:400;font-size:12px}
|
||||
table{width:100%;border-collapse:collapse;background:var(--panel);border:1px solid var(--line);border-radius:12px;overflow:hidden;font-size:13.5px}
|
||||
th{position:sticky;top:0;background:#0e141f;color:var(--dim);text-transform:uppercase;font-size:11px;letter-spacing:.06em;text-align:left;padding:9px 12px}
|
||||
td{padding:8px 12px;border-top:1px solid var(--line);max-width:340px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||||
tr:hover td{background:#151d2e}
|
||||
.badge{display:inline-block;padding:1px 9px;border-radius:99px;font-size:11.5px;font-weight:600}
|
||||
.b-NEW{background:#0e3a2d;color:var(--ok)}.b-DEAD{background:#3a1414;color:var(--bad)}.b-ERR{background:#3a2b0e;color:var(--warn)}
|
||||
.eng{display:grid;grid-template-columns:repeat(auto-fit,minmax(230px,1fr));gap:12px}
|
||||
.eng .card{display:flex;justify-content:space-between;align-items:center;gap:10px}
|
||||
.dot{width:10px;height:10px;border-radius:99px;display:inline-block;margin-right:7px}
|
||||
.dot.on{background:var(--ok);box-shadow:0 0 8px var(--ok)}
|
||||
.dot.off{background:#39445a}
|
||||
button{background:#1a2333;color:var(--fg);border:1px solid var(--line);border-radius:8px;padding:7px 14px;font-size:13px;cursor:pointer}
|
||||
button:hover{border-color:var(--acc);color:var(--acc)}
|
||||
button.stop:hover{border-color:var(--bad);color:var(--bad)}
|
||||
#log{background:#0a0d12;border:1px solid var(--line);border-radius:12px;padding:14px;font:12.5px/1.5 ui-monospace,Consolas,monospace;white-space:pre-wrap;color:#8fa3bd;max-height:320px;overflow:auto}
|
||||
.bar{height:7px;background:#1a2333;border-radius:99px;overflow:hidden;margin-top:8px}
|
||||
.bar i{display:block;height:100%;background:var(--acc)}
|
||||
.filters{display:flex;gap:8px;margin:10px 0;flex-wrap:wrap}
|
||||
.filters select,.filters input{background:var(--panel);border:1px solid var(--line);color:var(--fg);border-radius:8px;padding:7px 10px;font-size:13px}
|
||||
footer{margin-top:34px;color:var(--dim);font-size:12px}
|
||||
"""
|
||||
|
||||
_JS = """
|
||||
async function jget(u){const r=await fetch(u);return r.json();}
|
||||
async function post(u){const r=await fetch(u,{method:'POST'});return r.json();}
|
||||
|
||||
async function refresh(){
|
||||
try{
|
||||
const s=await jget('/api/stats');
|
||||
el('m-total').textContent=s.total.toLocaleString('ru');
|
||||
el('m-today').textContent=s.today.toLocaleString('ru');
|
||||
const alive=Object.values(s.engines).filter(e=>e.alive).length;
|
||||
el('m-eng').textContent=alive+' / '+Object.keys(s.engines).length;
|
||||
el('m-ts').textContent=s.ts;
|
||||
// движки
|
||||
const box=el('eng');box.innerHTML='';
|
||||
for(const [id,e] of Object.entries(s.engines)){
|
||||
const c=document.createElement('div');c.className='card';
|
||||
c.innerHTML=`<div><span class="dot ${e.alive?'on':'off'}"></span><b>${id}</b><div style="color:var(--dim);font-size:12px">${e.desc}${e.auto?' · auto':''}</div></div>`;
|
||||
const btn=document.createElement('button');
|
||||
btn.textContent=e.alive?'стоп':'старт';
|
||||
if(e.alive)btn.className='stop';
|
||||
btn.onclick=async()=>{e.alive?await post('/api/engine/'+id+'/stop'):await post('/api/engine/'+id+'/start');refresh();};
|
||||
c.appendChild(btn);box.appendChild(c);
|
||||
}
|
||||
// статусы
|
||||
const st=el('t-status');st.innerHTML='';
|
||||
const mx=Math.max(...s.by_status.map(x=>x.n),1);
|
||||
for(const r of s.by_status){
|
||||
const tr=document.createElement('tr');
|
||||
tr.innerHTML=`<td><span class="badge b-${r.status}">${r.status}</span></td><td>${r.n.toLocaleString('ru')}</td>
|
||||
<td style="width:40%"><div class="bar"><i style="width:${(100*r.n/mx).toFixed(0)}%"></i></div></td>`;
|
||||
st.appendChild(tr);
|
||||
}
|
||||
// провайдеры
|
||||
const pv=el('t-prov');pv.innerHTML='';
|
||||
for(const r of s.by_prov){
|
||||
const tr=document.createElement('tr');
|
||||
tr.innerHTML=`<td>${r.prov}</td><td>${r.n.toLocaleString('ru')}</td>`;
|
||||
tr.style.cursor='pointer';
|
||||
tr.onclick=()=>{el('f-prov').value=r.prov;loadKeys();};
|
||||
pv.appendChild(tr);
|
||||
}
|
||||
loadKeys();
|
||||
loadLog();
|
||||
}catch(e){console.error(e);}
|
||||
}
|
||||
|
||||
async function loadKeys(){
|
||||
const st=el('f-status').value,pv=el('f-prov').value,lim=el('f-limit').value||50;
|
||||
const d=await jget(`/api/keys?status=${st}&prov=${pv}&limit=${lim}`);
|
||||
const t=el('t-keys');t.innerHTML='';
|
||||
for(const r of d.rows){
|
||||
const tr=document.createElement('tr');
|
||||
tr.innerHTML=`<td><span class="badge b-${r.status}">${r.status}</span></td><td>${r.prov}</td>
|
||||
<td title="${r.val}">${r.val}</td><td title="${r.repo||''}">${r.repo||''}</td><td>${r.found||''}</td>`;
|
||||
t.appendChild(tr);
|
||||
}
|
||||
el('k-count').textContent=d.rows.length+' показано';
|
||||
}
|
||||
|
||||
async function loadLog(){
|
||||
const r=await fetch('/api/log?engine='+el('f-log').value);
|
||||
el('log').textContent=await r.text();
|
||||
el('log').scrollTop=el('log').scrollHeight;
|
||||
}
|
||||
|
||||
function el(id){return document.getElementById(id);}
|
||||
function exportKeys(){const st=el('f-status').value,pv=el('f-prov').value;
|
||||
window.open(`/api/export?status=${st}&prov=${pv}`,'_blank');}
|
||||
|
||||
refresh();
|
||||
setInterval(refresh,5000);
|
||||
"""
|
||||
|
||||
_HTML = f"""<!DOCTYPE html>
|
||||
<html lang="ru"><head><meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>KeyHunter Agent</title>
|
||||
<style>{_CSS}</style></head>
|
||||
<body><div class="wrap">
|
||||
<h1>Key<b>Hunter</b> Agent</h1>
|
||||
<div class="sub">автономный оператор парсер-фермы · GitHub / Bitbucket / Codeberg / Clojars / CRAN / crates.io / Go</div>
|
||||
|
||||
<div class="hero">
|
||||
<div class="card"><div class="k">Всего ключей</div><div class="v acc" id="m-total">—</div></div>
|
||||
<div class="card"><div class="k">Найдено сегодня</div><div class="v ok" id="m-today">—</div></div>
|
||||
<div class="card"><div class="k">Движки живы</div><div class="v" id="m-eng">—</div></div>
|
||||
<div class="card"><div class="k">Обновлено</div><div class="v" style="font-size:15px" id="m-ts">—</div></div>
|
||||
</div>
|
||||
|
||||
<h2>Движки <small>кнопки старт/стоп · авто = автопилот держит живым</small></h2>
|
||||
<div class="eng" id="eng"></div>
|
||||
|
||||
<h2>Статусы <small>NEW = не проверен</small></h2>
|
||||
<table><thead><tr><th>Статус</th><th>Ключей</th><th>Доля</th></tr></thead><tbody id="t-status"></tbody></table>
|
||||
|
||||
<h2>Провайдеры <small>клик — фильтр в таблице ниже</small></h2>
|
||||
<table><thead><tr><th>Провайдер</th><th>Ключей</th></tr></thead><tbody id="t-prov"></tbody></table>
|
||||
|
||||
<h2>Последние ключи <small id="k-count"></small></h2>
|
||||
<div class="filters">
|
||||
<select id="f-status"><option value="">любой статус</option><option>NEW</option><option>ALIVE</option><option>DEAD</option><option>ERR</option></select>
|
||||
<input id="f-prov" placeholder="провайдер, напр. OPENAI" style="width:170px">
|
||||
<select id="f-limit"><option>50</option><option>100</option><option>200</option></select>
|
||||
<button onclick="loadKeys()">обновить</button>
|
||||
<button onclick="exportKeys()">экспорт .txt</button>
|
||||
</div>
|
||||
<table><thead><tr><th>Статус</th><th>Пров</th><th>Ключ</th><th>Источник</th><th>Когда</th></tr></thead><tbody id="t-keys"></tbody></table>
|
||||
|
||||
<h2>Лог движка <small>живой хвост</small></h2>
|
||||
<div class="filters">
|
||||
<select id="f-log" onchange="loadLog()">""" + "".join(
|
||||
f'<option value="{k}">{k}</option>' for k in ENGINES) + f"""</select>
|
||||
</div>
|
||||
<div id="log"></div>
|
||||
|
||||
<footer>KeyHunter Agent · data/keys.db · автопилот раз в 60с · порт 8779</footer>
|
||||
<script>{_JS}</script>
|
||||
</div></body></html>"""
|
||||
|
||||
|
||||
@app.get("/dashboard", response_class=HTMLResponse)
|
||||
def dashboard():
|
||||
return _HTML
|
||||
|
||||
|
||||
@app.get("/", response_class=HTMLResponse)
|
||||
def root():
|
||||
return _HTML
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--port", type=int, default=8779)
|
||||
ap.add_argument("--host", default="127.0.0.1")
|
||||
ap.add_argument("--no-auto", action="store_true", help="не запускать автопилот")
|
||||
a = ap.parse_args()
|
||||
|
||||
if not a.no_auto:
|
||||
threading.Thread(target=autopilot_loop, daemon=True).start()
|
||||
|
||||
import uvicorn
|
||||
print(f"KEYHUNTER AGENT on http://{a.host}:{a.port}/dashboard", flush=True)
|
||||
uvicorn.run(app, host=a.host, port=a.port, log_level="warning")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user