381 lines
16 KiB
Python
381 lines
16 KiB
Python
"""KEYHUNTER AGENT — автономный оператор парсер-фермы (Apodex-стиль).
|
||
|
||
Отдельный софт: FastAPI :8779 + дашборд + автопилот движков.
|
||
Движки (gh_eternal, bitbucket, codeberg, clojars, cran, crates, goproxy)
|
||
запускаются как подпроцессы, статус в памяти, ключи в data/keys.db.
|
||
Запуск: python agent.py (или python agent.py --port 8779)
|
||
"""
|
||
import argparse
|
||
import os
|
||
import sqlite3
|
||
import subprocess
|
||
import sys
|
||
import threading
|
||
import time
|
||
from pathlib import Path
|
||
|
||
from fastapi import FastAPI
|
||
from fastapi.responses import HTMLResponse, PlainTextResponse
|
||
|
||
ROOT = Path(__file__).resolve().parent
|
||
DB = ROOT / "data" / "keys.db"
|
||
|
||
# ---------------------------------------------------------------- движки
|
||
ENGINES = {
|
||
"gh": {"script": "gh_eternal.py", "log": "gh_eternal.log", "desc": "GitHub dork-hunt 787"},
|
||
"bitbucket": {"script": "bitbucket_eternal.py", "log": "bitbucket_eternal.log", "desc": "Bitbucket repos"},
|
||
"codeberg": {"script": "codeberg_eternal.py", "log": "codeberg_eternal.log", "desc": "Codeberg repos"},
|
||
"clojars": {"script": "clojars_eternal.py", "log": "clojars_eternal.log", "desc": "Clojars (maven)"},
|
||
"cran": {"script": "cran_eternal.py", "log": "cran_eternal.log", "desc": "CRAN packages"},
|
||
"crates": {"script": "crates_eternal.py", "log": "crates_eternal.log", "desc": "crates.io"},
|
||
"goproxy": {"script": "goproxy_eternal.py", "log": "goproxy_eternal.log", "desc": "Go proxy"},
|
||
}
|
||
|
||
_proc: dict = {} # engine_id -> Popen
|
||
_proc_lock = threading.Lock()
|
||
AUTOS = {"gh", "crates", "goproxy"} # автопилот: держим живыми
|
||
|
||
|
||
def _spawn(eid: str) -> bool:
|
||
eng = ENGINES.get(eid)
|
||
if not eng or not (ROOT / eng["script"]).exists():
|
||
return False
|
||
with _proc_lock:
|
||
p = _proc.get(eid)
|
||
if p and p.poll() is None:
|
||
return True
|
||
# ponytail: CREATE_NEW_PROCESS_GROUP -> корректный terminate на Windows
|
||
_proc[eid] = subprocess.Popen(
|
||
[sys.executable, str(ROOT / eng["script"])],
|
||
cwd=str(ROOT),
|
||
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
||
creationflags=subprocess.CREATE_NEW_PROCESS_GROUP if os.name == "nt" else 0,
|
||
)
|
||
return True
|
||
|
||
|
||
def _stop(eid: str) -> bool:
|
||
with _proc_lock:
|
||
p = _proc.pop(eid, None)
|
||
if not p:
|
||
return False
|
||
try:
|
||
p.terminate()
|
||
p.wait(timeout=8)
|
||
except Exception:
|
||
try: p.kill()
|
||
except Exception: pass
|
||
return True
|
||
|
||
|
||
def engines_alive() -> dict:
|
||
with _proc_lock:
|
||
items = dict(_proc)
|
||
return {k: (p.poll() is None) for k, p in items.items()}
|
||
|
||
|
||
def autopilot_loop():
|
||
"""Раз в минуту: поднимать движки из AUTOS, мёртвые рестартить."""
|
||
while True:
|
||
for eid in AUTOS:
|
||
alive = engines_alive().get(eid, False)
|
||
if not alive:
|
||
_spawn(eid)
|
||
time.sleep(60)
|
||
|
||
|
||
# ---------------------------------------------------------------- db
|
||
def q(sql: str, args=()):
|
||
con = sqlite3.connect(f"file:{DB}?mode=ro", uri=True, timeout=30)
|
||
try:
|
||
con.row_factory = sqlite3.Row
|
||
return [dict(r) for r in con.execute(sql, args).fetchall()]
|
||
finally:
|
||
con.close()
|
||
|
||
|
||
def stats() -> dict:
|
||
total = q("SELECT COUNT(*) n FROM keys")[0]["n"]
|
||
by_status = q("SELECT status, COUNT(*) n FROM keys GROUP BY status ORDER BY n DESC")
|
||
by_prov = q("SELECT prov, COUNT(*) n FROM keys GROUP BY prov ORDER BY n DESC LIMIT 20")
|
||
today = q(
|
||
"SELECT COUNT(*) n FROM keys WHERE found LIKE ?",
|
||
(time.strftime("%Y-%m-%d") + "%",),
|
||
)[0]["n"]
|
||
alive = engines_alive()
|
||
return {
|
||
"total": total,
|
||
"today": today,
|
||
"by_status": by_status,
|
||
"by_prov": by_prov,
|
||
"engines": {k: {"alive": alive.get(k, False),
|
||
"auto": k in AUTOS,
|
||
"desc": v["desc"]} for k, v in ENGINES.items()},
|
||
"ts": time.strftime("%Y-%m-%d %H:%M:%S"),
|
||
}
|
||
|
||
|
||
def tail(path: Path, n=80) -> str:
|
||
try:
|
||
with open(path, "rb") as f:
|
||
f.seek(0, 2)
|
||
size = f.tell()
|
||
f.seek(max(0, size - 200_000))
|
||
chunk = f.read().decode("utf-8", errors="replace")
|
||
return "\n".join(chunk.splitlines()[-n:])
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
# ---------------------------------------------------------------- api
|
||
app = FastAPI(title="KeyHunter Agent")
|
||
|
||
|
||
@app.get("/api/stats")
|
||
def api_stats():
|
||
return stats()
|
||
|
||
|
||
@app.get("/api/keys")
|
||
def api_keys(status: str = "", prov: str = "", limit: int = 50, offset: int = 0):
|
||
limit = max(1, min(limit, 500))
|
||
sql, args = "SELECT id, prov, status, plan, price, remaining, found, repo, val FROM keys", []
|
||
w = []
|
||
if status:
|
||
w.append("status=?"); args.append(status)
|
||
if prov:
|
||
w.append("prov=?"); args.append(prov)
|
||
if w:
|
||
sql += " WHERE " + " AND ".join(w)
|
||
sql += " ORDER BY id DESC LIMIT ? OFFSET ?"
|
||
rows = q(sql, (*args, limit, offset))
|
||
for r in rows: # маскируем ключи в выдаче
|
||
v = r.get("val") or ""
|
||
r["val"] = (v[:6] + "…" + v[-4:]) if len(v) > 14 else v
|
||
return {"rows": rows, "offset": offset, "limit": limit}
|
||
|
||
|
||
@app.get("/api/log")
|
||
def api_log(engine: str = "gh", n: int = 80):
|
||
eng = ENGINES.get(engine)
|
||
path = ROOT / (eng["log"] if eng else "gh_eternal.log")
|
||
return PlainTextResponse(tail(Path(path), min(n, 400)) or "(лог пуст)")
|
||
|
||
|
||
@app.post("/api/engine/{eid}/start")
|
||
async def api_start(eid: str):
|
||
ok = _spawn(eid)
|
||
return {"ok": ok, "alive": engines_alive().get(eid, False)}
|
||
|
||
|
||
@app.post("/api/engine/{eid}/stop")
|
||
async def api_stop(eid: str):
|
||
ok = _stop(eid)
|
||
return {"ok": ok}
|
||
|
||
|
||
@app.get("/api/export")
|
||
def api_export(status: str = "NEW", prov: str = ""):
|
||
sql, args = "SELECT val, prov, status, repo FROM keys", []
|
||
w = []
|
||
if status:
|
||
w.append("status=?"); args.append(status)
|
||
if prov:
|
||
w.append("prov=?"); args.append(prov)
|
||
if w:
|
||
sql += " WHERE " + " AND ".join(w)
|
||
body = "\n".join(f"{r['val']}\t{r['prov']}\t{r['repo']}" for r in q(sql, args))
|
||
return PlainTextResponse(body)
|
||
|
||
|
||
@app.get("/health")
|
||
def health():
|
||
return {"ok": True, "engines": engines_alive()}
|
||
|
||
|
||
# ---------------------------------------------------------------- dashboard
|
||
_CSS = """
|
||
:root{--bg:#0b0e14;--panel:#121826;--line:#1e2839;--fg:#c9d4e3;--dim:#5d6b80;--acc:#38bdf8;--ok:#34d399;--warn:#fbbf24;--bad:#f87171}
|
||
*{margin:0;padding:0;box-sizing:border-box}
|
||
body{background:var(--bg);color:var(--fg);font:15px/1.55 ui-sans-serif,system-ui,'Segoe UI',Roboto,sans-serif}
|
||
.wrap{max-width:1080px;margin:0 auto;padding:28px 20px 60px}
|
||
h1{font-size:26px;letter-spacing:-.02em}
|
||
h1 b{color:var(--acc)}
|
||
.sub{color:var(--dim);margin:6px 0 22px}
|
||
.hero{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin-bottom:24px}
|
||
.card{background:var(--panel);border:1px solid var(--line);border-radius:12px;padding:16px 18px}
|
||
.card .k{font-size:12px;text-transform:uppercase;letter-spacing:.08em;color:var(--dim)}
|
||
.card .v{font-size:26px;font-weight:700;margin-top:4px}
|
||
.card .v.acc{color:var(--acc)}.card .v.ok{color:var(--ok)}
|
||
h2{font-size:17px;margin:28px 0 12px;display:flex;gap:10px;align-items:baseline}
|
||
h2 small{color:var(--dim);font-weight:400;font-size:12px}
|
||
table{width:100%;border-collapse:collapse;background:var(--panel);border:1px solid var(--line);border-radius:12px;overflow:hidden;font-size:13.5px}
|
||
th{position:sticky;top:0;background:#0e141f;color:var(--dim);text-transform:uppercase;font-size:11px;letter-spacing:.06em;text-align:left;padding:9px 12px}
|
||
td{padding:8px 12px;border-top:1px solid var(--line);max-width:340px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
||
tr:hover td{background:#151d2e}
|
||
.badge{display:inline-block;padding:1px 9px;border-radius:99px;font-size:11.5px;font-weight:600}
|
||
.b-NEW{background:#0e3a2d;color:var(--ok)}.b-DEAD{background:#3a1414;color:var(--bad)}.b-ERR{background:#3a2b0e;color:var(--warn)}
|
||
.eng{display:grid;grid-template-columns:repeat(auto-fit,minmax(230px,1fr));gap:12px}
|
||
.eng .card{display:flex;justify-content:space-between;align-items:center;gap:10px}
|
||
.dot{width:10px;height:10px;border-radius:99px;display:inline-block;margin-right:7px}
|
||
.dot.on{background:var(--ok);box-shadow:0 0 8px var(--ok)}
|
||
.dot.off{background:#39445a}
|
||
button{background:#1a2333;color:var(--fg);border:1px solid var(--line);border-radius:8px;padding:7px 14px;font-size:13px;cursor:pointer}
|
||
button:hover{border-color:var(--acc);color:var(--acc)}
|
||
button.stop:hover{border-color:var(--bad);color:var(--bad)}
|
||
#log{background:#0a0d12;border:1px solid var(--line);border-radius:12px;padding:14px;font:12.5px/1.5 ui-monospace,Consolas,monospace;white-space:pre-wrap;color:#8fa3bd;max-height:320px;overflow:auto}
|
||
.bar{height:7px;background:#1a2333;border-radius:99px;overflow:hidden;margin-top:8px}
|
||
.bar i{display:block;height:100%;background:var(--acc)}
|
||
.filters{display:flex;gap:8px;margin:10px 0;flex-wrap:wrap}
|
||
.filters select,.filters input{background:var(--panel);border:1px solid var(--line);color:var(--fg);border-radius:8px;padding:7px 10px;font-size:13px}
|
||
footer{margin-top:34px;color:var(--dim);font-size:12px}
|
||
"""
|
||
|
||
_JS = """
|
||
async function jget(u){const r=await fetch(u);return r.json();}
|
||
async function post(u){const r=await fetch(u,{method:'POST'});return r.json();}
|
||
|
||
async function refresh(){
|
||
try{
|
||
const s=await jget('/api/stats');
|
||
el('m-total').textContent=s.total.toLocaleString('ru');
|
||
el('m-today').textContent=s.today.toLocaleString('ru');
|
||
const alive=Object.values(s.engines).filter(e=>e.alive).length;
|
||
el('m-eng').textContent=alive+' / '+Object.keys(s.engines).length;
|
||
el('m-ts').textContent=s.ts;
|
||
// движки
|
||
const box=el('eng');box.innerHTML='';
|
||
for(const [id,e] of Object.entries(s.engines)){
|
||
const c=document.createElement('div');c.className='card';
|
||
c.innerHTML=`<div><span class="dot ${e.alive?'on':'off'}"></span><b>${id}</b><div style="color:var(--dim);font-size:12px">${e.desc}${e.auto?' · auto':''}</div></div>`;
|
||
const btn=document.createElement('button');
|
||
btn.textContent=e.alive?'стоп':'старт';
|
||
if(e.alive)btn.className='stop';
|
||
btn.onclick=async()=>{e.alive?await post('/api/engine/'+id+'/stop'):await post('/api/engine/'+id+'/start');refresh();};
|
||
c.appendChild(btn);box.appendChild(c);
|
||
}
|
||
// статусы
|
||
const st=el('t-status');st.innerHTML='';
|
||
const mx=Math.max(...s.by_status.map(x=>x.n),1);
|
||
for(const r of s.by_status){
|
||
const tr=document.createElement('tr');
|
||
tr.innerHTML=`<td><span class="badge b-${r.status}">${r.status}</span></td><td>${r.n.toLocaleString('ru')}</td>
|
||
<td style="width:40%"><div class="bar"><i style="width:${(100*r.n/mx).toFixed(0)}%"></i></div></td>`;
|
||
st.appendChild(tr);
|
||
}
|
||
// провайдеры
|
||
const pv=el('t-prov');pv.innerHTML='';
|
||
for(const r of s.by_prov){
|
||
const tr=document.createElement('tr');
|
||
tr.innerHTML=`<td>${r.prov}</td><td>${r.n.toLocaleString('ru')}</td>`;
|
||
tr.style.cursor='pointer';
|
||
tr.onclick=()=>{el('f-prov').value=r.prov;loadKeys();};
|
||
pv.appendChild(tr);
|
||
}
|
||
loadKeys();
|
||
loadLog();
|
||
}catch(e){console.error(e);}
|
||
}
|
||
|
||
async function loadKeys(){
|
||
const st=el('f-status').value,pv=el('f-prov').value,lim=el('f-limit').value||50;
|
||
const d=await jget(`/api/keys?status=${st}&prov=${pv}&limit=${lim}`);
|
||
const t=el('t-keys');t.innerHTML='';
|
||
for(const r of d.rows){
|
||
const tr=document.createElement('tr');
|
||
tr.innerHTML=`<td><span class="badge b-${r.status}">${r.status}</span></td><td>${r.prov}</td>
|
||
<td title="${r.val}">${r.val}</td><td title="${r.repo||''}">${r.repo||''}</td><td>${r.found||''}</td>`;
|
||
t.appendChild(tr);
|
||
}
|
||
el('k-count').textContent=d.rows.length+' показано';
|
||
}
|
||
|
||
async function loadLog(){
|
||
const r=await fetch('/api/log?engine='+el('f-log').value);
|
||
el('log').textContent=await r.text();
|
||
el('log').scrollTop=el('log').scrollHeight;
|
||
}
|
||
|
||
function el(id){return document.getElementById(id);}
|
||
function exportKeys(){const st=el('f-status').value,pv=el('f-prov').value;
|
||
window.open(`/api/export?status=${st}&prov=${pv}`,'_blank');}
|
||
|
||
refresh();
|
||
setInterval(refresh,5000);
|
||
"""
|
||
|
||
_HTML = f"""<!DOCTYPE html>
|
||
<html lang="ru"><head><meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||
<title>KeyHunter Agent</title>
|
||
<style>{_CSS}</style></head>
|
||
<body><div class="wrap">
|
||
<h1>Key<b>Hunter</b> Agent</h1>
|
||
<div class="sub">автономный оператор парсер-фермы · GitHub / Bitbucket / Codeberg / Clojars / CRAN / crates.io / Go</div>
|
||
|
||
<div class="hero">
|
||
<div class="card"><div class="k">Всего ключей</div><div class="v acc" id="m-total">—</div></div>
|
||
<div class="card"><div class="k">Найдено сегодня</div><div class="v ok" id="m-today">—</div></div>
|
||
<div class="card"><div class="k">Движки живы</div><div class="v" id="m-eng">—</div></div>
|
||
<div class="card"><div class="k">Обновлено</div><div class="v" style="font-size:15px" id="m-ts">—</div></div>
|
||
</div>
|
||
|
||
<h2>Движки <small>кнопки старт/стоп · авто = автопилот держит живым</small></h2>
|
||
<div class="eng" id="eng"></div>
|
||
|
||
<h2>Статусы <small>NEW = не проверен</small></h2>
|
||
<table><thead><tr><th>Статус</th><th>Ключей</th><th>Доля</th></tr></thead><tbody id="t-status"></tbody></table>
|
||
|
||
<h2>Провайдеры <small>клик — фильтр в таблице ниже</small></h2>
|
||
<table><thead><tr><th>Провайдер</th><th>Ключей</th></tr></thead><tbody id="t-prov"></tbody></table>
|
||
|
||
<h2>Последние ключи <small id="k-count"></small></h2>
|
||
<div class="filters">
|
||
<select id="f-status"><option value="">любой статус</option><option>NEW</option><option>ALIVE</option><option>DEAD</option><option>ERR</option></select>
|
||
<input id="f-prov" placeholder="провайдер, напр. OPENAI" style="width:170px">
|
||
<select id="f-limit"><option>50</option><option>100</option><option>200</option></select>
|
||
<button onclick="loadKeys()">обновить</button>
|
||
<button onclick="exportKeys()">экспорт .txt</button>
|
||
</div>
|
||
<table><thead><tr><th>Статус</th><th>Пров</th><th>Ключ</th><th>Источник</th><th>Когда</th></tr></thead><tbody id="t-keys"></tbody></table>
|
||
|
||
<h2>Лог движка <small>живой хвост</small></h2>
|
||
<div class="filters">
|
||
<select id="f-log" onchange="loadLog()">""" + "".join(
|
||
f'<option value="{k}">{k}</option>' for k in ENGINES) + f"""</select>
|
||
</div>
|
||
<div id="log"></div>
|
||
|
||
<footer>KeyHunter Agent · data/keys.db · автопилот раз в 60с · порт 8779</footer>
|
||
<script>{_JS}</script>
|
||
</div></body></html>"""
|
||
|
||
|
||
@app.get("/dashboard", response_class=HTMLResponse)
|
||
def dashboard():
|
||
return _HTML
|
||
|
||
|
||
@app.get("/", response_class=HTMLResponse)
|
||
def root():
|
||
return _HTML
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--port", type=int, default=8779)
|
||
ap.add_argument("--host", default="127.0.0.1")
|
||
ap.add_argument("--no-auto", action="store_true", help="не запускать автопилот")
|
||
a = ap.parse_args()
|
||
|
||
if not a.no_auto:
|
||
threading.Thread(target=autopilot_loop, daemon=True).start()
|
||
|
||
import uvicorn
|
||
print(f"KEYHUNTER AGENT on http://{a.host}:{a.port}/dashboard", flush=True)
|
||
uvicorn.run(app, host=a.host, port=a.port, log_level="warning")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|