parser: ETag/304-кэш, 61 авто-источник из 81 кандидата (+RSS AI/security), правила farm/weights, тесты sources_auto

tools/auto_expand.py

@@ -27,6 +27,8 @@ CAND = ROOT / "data" / "source_candidates.json"
 HEALTH = ROOT / "data" / "source_health.json"
 OUT = ROOT / "omni" / "sources_auto.py"
 PUBLIC_APIS = "https://raw.githubusercontent.com/public-apis/public-apis/master/README.md"
+RE_URL = __import__("re").compile(r"\((https?://[^)\s]+)\)")
+RE_LINK = __import__("re").compile(r"\[([^\]]+)\]\([^)]+\)")
 
 HEADER = '''"""Авто-расширение: источники на публичных API без ключа.
 
@@ -144,25 +146,48 @@ def write_module(cands, results):
     print("\nwrote %s: %d live of %d" % (OUT.name, live, len(cands)))
 
 
+CAT_OK = ("Artificial Intelligence", "Machine Learning", "Security", "News", "Open Data",
+          "Development", "Text Analysis", "Data Validation", "Sports", "Cryptocurrency")
+KEEP_ANY_CAT = ("llm", "gpt", "openai", "threat", "cve", "vuln", "leak", "hack", "malware",
+                "phish", "proxy", "temp mail", "sms", "telegram", "scrap", "dns", "whois")
+BAD = ("waifu", "anime", "cat fact", "dog", "joke", "meme", "pokemon", "quote", "trivia",
+       "recipe", "cocktail", "game", "3d", "texture", "insurance", "crypto price")
+KW_OK = ("ai", "llm", "gpt", "model", "secur", "threat", "cve", "vuln", "leak", "hack", "news",
+         "feed", "telegram", "proxy", "mail", "sms", "text", "translate", "scrap", "search",
+         "crypto", "token", "domain", "dns", "ip", "geo")
+
+
 def discover(limit=40):
-    """Добрать бесплатные (Auth=No) API из public-apis."""
+    """Добрать бесплатные (Auth=No) API из public-apis: только релевантные категории/слова."""
     req = urllib.request.Request(PUBLIC_APIS, headers={"User-Agent": "omni-parse/1.0"})
     md = urllib.request.urlopen(req, timeout=40).read().decode("utf-8", "replace")
-    rows = []
+    rows, cat = [], ""
     for line in md.splitlines():
+        if line.startswith("### "):
+            cat = line[4:].strip()
+            continue
         if not line.startswith("|") or "http" not in line:
             continue
         cells = [c.strip() for c in line.strip("|").split("|")]
-        if len(cells) < 6:
+        if len(cells) < 5:
             continue
-        url, auth, https = cells[2], cells[4], cells[5]
-        if "No" not in auth or "http" not in url:
+        m = RE_URL.search(cells[0])
+        if not m:
             continue
-        if any(b in url for b in ("github.com/public-apis", "localhost")):
+        url = m.group(1)
+        title = RE_LINK.sub(r"\1", cells[0]).strip()
+        desc, auth = cells[1], cells[2]
+        if "No" not in auth:
             continue
-        rows.append({"name": "dis_" + url.split("//")[-1].split("/")[0].replace("www.", "").replace(".", "_")[:24],
-                     "url": url, "adapter": "generic_json", "kind": "json",
-                     "tags": ["abuse", "public_api"], "group": "json", "limit": 15})
+        blob = (title + " " + desc).lower()
+        if cat not in CAT_OK and not any(k in blob for k in KEEP_ANY_CAT):
+            continue
+        if any(b in blob for b in BAD):
+            continue
+        host = url.split("//")[-1].split("/")[0].replace("www.", "").replace(".", "_").replace("-", "_")
+        rows.append({"name": "dis_" + host[:26], "url": url, "adapter": "generic_json",
+                     "kind": "json", "tags": ["abuse", "public_api"], "group": "json", "limit": 15,
+                     "note": (title + " — " + desc)[:120], "cat": cat})
     uniq = {}
     for r in rows:
         uniq.setdefault(r["name"], r)
@@ -172,6 +197,8 @@ def discover(limit=40):
     cands.extend(new)
     save(CAND, cands)
     print("discover: +%d кандидатов (всего %d)" % (len(new), len(cands)))
+    for r in new[:12]:
+        print("  + %-30s %s" % (r["name"], r["note"][:80]))
     return new