119 lines
4.0 KiB
Python
119 lines
4.0 KiB
Python
"""Scan GitHub wikis of repo-hits: code search does NOT index wiki pages.
|
|
Wiki = separate git repo {owner}/{repo}.wiki.git — clone, grep PATTERNS."""
|
|
import os
|
|
KF_BASE = os.path.dirname(os.path.abspath(__file__))
|
|
os.makedirs(os.path.join(KF_BASE, "data"), exist_ok=True)
|
|
import shutil
|
|
import sqlite3
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import time
|
|
from concurrent.futures import ThreadPoolExecutor, as_completed
|
|
|
|
sys.path.insert(0, KF_BASE)
|
|
from gh_eternal import PATTERNS, GARBAGE, POOL # noqa: E402
|
|
|
|
DB = os.environ.get("KF_DB") or os.path.join(KF_BASE, "data/keys.db")
|
|
LOG = os.path.join(KF_BASE, "wiki_scan.log")
|
|
|
|
|
|
def log(msg):
|
|
print(f"[{time.strftime('%H:%M:%S')}] {msg}", flush=True)
|
|
|
|
|
|
def wiki_urls_from_db(limit):
|
|
db = sqlite3.connect(DB, timeout=60)
|
|
c = db.cursor()
|
|
c.execute("SELECT repo FROM keys WHERE repo IS NOT NULL AND repo != '' "
|
|
"GROUP BY repo ORDER BY count(*) DESC LIMIT ?", (limit,))
|
|
repos = []
|
|
for (r,) in c.fetchall():
|
|
repo = r.split("/")[0] + "/" + r.split("/")[1] if "/" in r else ""
|
|
if repo and repo not in repos:
|
|
repos.append(repo)
|
|
db.close()
|
|
return repos
|
|
|
|
|
|
def clone_and_scan(repo):
|
|
"""Clone wiki repo shallow, grep PATTERNS over all files."""
|
|
tok = POOL.tokens[0] if POOL.tokens else ""
|
|
url = f"https://x-access-token:{tok}@github.com/{repo}.wiki.git"
|
|
tmp = tempfile.mkdtemp(prefix="wiki_")
|
|
found = []
|
|
try:
|
|
r = subprocess.run(
|
|
["git", "clone", "--depth", "1", "-q", url, tmp],
|
|
capture_output=True, timeout=30)
|
|
if r.returncode != 0:
|
|
return repo, "no-wiki", []
|
|
import re
|
|
texts = []
|
|
import pathlib
|
|
for f in pathlib.Path(tmp).rglob("*"):
|
|
if f.is_file() and f.stat().st_size < 1_000_000:
|
|
try:
|
|
texts.append(f.read_text(errors="ignore"))
|
|
except Exception:
|
|
pass
|
|
blob = "\n".join(texts)
|
|
for prov, rx in PATTERNS:
|
|
for key in rx.findall(blob):
|
|
if isinstance(key, tuple):
|
|
key = key[0]
|
|
if any(g in key for g in GARBAGE) or len(key) < 30:
|
|
continue
|
|
found.append((prov, key))
|
|
return repo, f"scanned ({len(texts)} files)", found
|
|
except subprocess.TimeoutExpired:
|
|
return repo, "timeout", []
|
|
except Exception as e:
|
|
return repo, f"error {e!r}", []
|
|
finally:
|
|
shutil.rmtree(tmp, ignore_errors=True)
|
|
|
|
|
|
def main(limit, threads):
|
|
repos = wiki_urls_from_db(limit)
|
|
log(f"start: {len(repos)} repos to wiki-scan, {threads} threads")
|
|
db = sqlite3.connect(DB, timeout=120, check_same_thread=False)
|
|
cur = db.cursor()
|
|
seen = {r[0] for r in cur.execute("SELECT val FROM keys").fetchall()}
|
|
new_total = 0
|
|
lock_cur = __import__("threading").Lock()
|
|
|
|
def scan(repo):
|
|
nonlocal new_total
|
|
repo, status, found = clone_and_scan(repo)
|
|
n = 0
|
|
for prov, key in found:
|
|
if key in seen:
|
|
continue
|
|
with lock_cur:
|
|
seen.add(key)
|
|
import hashlib
|
|
cur.execute(
|
|
"INSERT OR IGNORE INTO keys "
|
|
"(hash, val, prov, status, found, repo) VALUES (?,?,?,?,?,?)",
|
|
(hashlib.sha1(key.encode()).hexdigest()[:16], key, prov,
|
|
"NEW", "wiki-scan", f"{repo}/wiki"))
|
|
db.commit()
|
|
new_total += 1
|
|
n += 1
|
|
if n or "scanned" in status:
|
|
log(f"{repo}: {status}, +{n} new")
|
|
return n
|
|
|
|
with ThreadPoolExecutor(max_workers=threads) as ex:
|
|
futs = [ex.submit(scan, r) for r in repos]
|
|
for _ in as_completed(futs):
|
|
pass
|
|
log(f"done: +{new_total} new keys")
|
|
db.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
limit = int(sys.argv[1]) if len(sys.argv) > 1 else 300
|
|
threads = int(sys.argv[2]) if len(sys.argv) > 2 else 8
|
|
main(limit, threads) |