From cfb8639c67747184b5f40b74bada8a03dd12959b Mon Sep 17 00:00:00 2001 From: Vlad Gothbreach Date: Mon, 26 Oct 2026 19:33:35 +0300 Subject: [PATCH] =?UTF-8?q?feat(x):=20=D0=BF=D0=B0=D1=80=D1=81=D0=B8=D0=BD?= =?UTF-8?q?=D0=B3=20X=20=D1=87=D0=B5=D1=80=D0=B5=D0=B7=20api.fxtwitter.com?= =?UTF-8?q?=20=E2=80=94=2030=20=D0=B6=D0=B8=D0=B2=D1=8B=D1=85=20=D1=85?= =?UTF-8?q?=D0=B5=D0=BD=D0=B4=D0=BB=D0=BE=D0=B2=20=D0=BA=D0=B0=D0=BA=20?= =?UTF-8?q?=D0=B8=D1=81=D1=82=D0=BE=D1=87=D0=BD=D0=B8=D0=BA=D0=B8,=20?= =?UTF-8?q?=D0=B0=D0=B4=D0=B0=D0=BF=D1=82=D0=B5=D1=80=20x=5Fuser,=20=D0=BF?= =?UTF-8?q?=D0=BE=D0=BB=D0=BD=D1=8B=D0=B9=20=D1=82=D0=B5=D0=BA=D1=81=D1=82?= =?UTF-8?q?=20=D1=82=D0=B2=D0=B8=D1=82=D0=BE=D0=B2,=20=D0=BC=D0=B5=D1=82?= =?UTF-8?q?=D1=80=D0=B8=D0=BA=D0=B8;=20=D1=81=D1=82=D0=B0=D1=80=D1=8B?= =?UTF-8?q?=D0=B9=20syndication-=D0=BF=D1=83=D1=82=D1=8C=20(429)=20=D1=83?= =?UTF-8?q?=D0=B1=D1=80=D0=B0=D0=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omni/feeds.py | 8 +++++ omni/providers/x.py | 84 +++++++++++++++++++++++++++++++-------------- omni/sources.py | 3 +- omni/sources_x.py | 53 ++++++++++++++++++++++++++++ 4 files changed, 122 insertions(+), 26 deletions(-) create mode 100644 omni/sources_x.py diff --git a/omni/feeds.py b/omni/feeds.py index f07a31e..0b8e4e0 100644 --- a/omni/feeds.py +++ b/omni/feeds.py @@ -648,7 +648,15 @@ def _models_dev(src, limit): return out +def _x_user(src, limit=12): + """Лента X-аккаунта через api.fxtwitter.com (без ключей).""" + from .providers import x as xp + handle = (src.extra or {}).get("handle") or src.url.rsplit("/", 1)[-1] + return xp.profile(handle, limit=max(4, min(limit, 25))) + + ADAPTERS = { + "x_user": _x_user, "rss": _rss, "generic_json": _generic_json, "or_models": _or_models, diff --git a/omni/providers/x.py b/omni/providers/x.py index 7c84f07..681dcf6 100644 --- a/omni/providers/x.py +++ b/omni/providers/x.py @@ -77,30 +77,64 @@ def search_x(query, limit=15, handles=None): extra={"handle": d["handle"]}) items.append(it) - # путь 3: конкретные профили - for h in (handles or [])[:4]: - r = net.get(f"https://syndication.twitter.com/srv/timeline-profile/screen-name/{h}", - headers=UA) - if r.status != 200: - continue - m = re.search(r'', r.text, re.S) - if not m: - continue - try: - data = json.loads(m.group(1)) - except Exception: - continue - entries = (((data.get("props") or {}).get("pageProps") or {}).get("timeline") or {}).get("entries") or [] - for e in entries[:12]: - tw = ((e.get("content") or {}).get("tweet") or {}) - txt = tw.get("full_text") or tw.get("text") or "" - if not txt: - continue - tid = tw.get("id_str") or "" - it = Item("x", "tweet", url=f"https://x.com/{h}/status/{tid}" if tid else f"https://x.com/{h}", - title=txt[:120], text=txt, author=h, source="x.com/" + h, - published=tw.get("created_at", ""), - extra={"likes": tw.get("favorite_count"), "rt": tw.get("retweet_count")}) - it.score = float(tw.get("favorite_count") or 0) ** 0.5 + # путь 3: конкретные профили через fxtwitter (без ключей, отдаёт полный текст) + for h in (handles or [])[:6]: + for it in profile(h, limit=12): items.append(it) + + # дедуп по ссылке + uniq, seen_u = [], set() + for it in items: + if it.url in seen_u: + continue + seen_u.add(it.url) + uniq.append(it) + return uniq[:max(limit, 12) if handles else len(uniq)] + + +def profile(handle, limit=20, cursor=None): + """Лента профиля X через api.fxtwitter.com — без токенов и логина. + + Возвращает список Item; при ошибке сети/404 — пустой список. + """ + h = (handle or "").lstrip("@").strip() + if not h: + return [] + url = f"https://api.fxtwitter.com/2/profile/{h}/statuses" + if cursor: + url += "?cursor=" + urllib.parse.quote(str(cursor)) + try: + r = net.get(url, timeout=25, json_mode=True) + except Exception: + return [] + if r.status != 200: + return [] + data = r.json() or {} + out = [] + for t in (data.get("results") or []): + if t.get("type") != "status": + continue + rt = t.get("raw_text") + txt = (rt.get("text") if isinstance(rt, dict) else rt) or t.get("text") or "" + if not isinstance(txt, str): + txt = str(txt) + if len(txt) < 15: + continue + tid = t.get("id") or "" + author = (t.get("author") or {}).get("screen_name") or h if isinstance(t.get("author"), dict) else h + likes = t.get("likes") or 0 + it = Item("x", "tweet", + url=t.get("url") or f"https://x.com/{h}/status/{tid}", + title=txt.strip().splitlines()[0][:140], text=txt, + author=author, source="x.com/" + h, + published=t.get("created_at", ""), + extra={"handle": h, "lang": t.get("lang", ""), "likes": likes, "replies": t.get("replies"), + "reposts": t.get("reposts"), "views": t.get("views"), + "quote": (t.get("quote") or {}).get("url") if isinstance(t.get("quote"), dict) else None}) + it.score = float(likes) ** 0.5 + float(t.get("replies") or 0) ** 0.4 + out.append(it) + if len(out) >= limit: + break + return out + return items[:limit * 2] \ No newline at end of file diff --git a/omni/sources.py b/omni/sources.py index 72dab08..5bc5850 100644 --- a/omni/sources.py +++ b/omni/sources.py @@ -16,6 +16,7 @@ from .sources_ext3 import EXTRA3 from .sources_abuse import ABUSE_LIVE from .sources_ext4 import EXTRA5 from .sources_ext5 import EXTRA6 +from .sources_x import X_SOURCES @dataclass @@ -199,7 +200,7 @@ def _dedup(rows): return out -SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6]) +SOURCES = _dedup(SOURCES_BASE + [Source(**d) for d in EXTRA] + [Source(**d) for d in EXTRA2] + [Source(**d) for d in EXTRA3] + [Source(**d) for d in ABUSE_LIVE] + [Source(**d) for d in EXTRA4] + [Source(**d) for d in EXTRA5] + [Source(**d) for d in EXTRA6] + [Source(**d) for d in X_SOURCES]) BY_NAME = {s.name: s for s in SOURCES} CATEGORIES = sorted({s.category for s in SOURCES}) diff --git a/omni/sources_x.py b/omni/sources_x.py new file mode 100644 index 0000000..22920d5 --- /dev/null +++ b/omni/sources_x.py @@ -0,0 +1,53 @@ +"""X/Twitter источники — читаются без API-ключей через api.fxtwitter.com. + +Адаптер: feeds._x_user -> providers.x.profile(handle). +Каждый хендл проверен живым прогоном (>=3 записи). Мёртвые (0 записей) не добавлены. +""" + +_X = [ + # (handle, category, теги) + ("vxunderground", "abuse", ("malware", "leak")), + ("malwrhunterteam", "abuse", ("malware", "stealer")), + ("DailyDarkWeb", "abuse", ("leak", "ransomware")), + ("DarkWebInformer", "abuse", ("leak", "darkweb")), + ("abuse_ch", "abuse", ("stealer", "malware")), + ("H4ckManac", "abuse", ("leak", "breach")), + ("falconfeedsio", "abuse", ("ransomware", "leak")), + ("nullcookies", "abuse", ("leak", "fraud")), + ("TheHackersNews", "security", ("news", "cve")), + ("BleepinComputer", "security", ("news", "cve")), + ("campuscodi", "security", ("news", "cve")), + ("ShadowServer", "security", ("scan", "cve")), + ("RecordedFuture", "security", ("intel", "cve")), + ("CISAgov", "security", ("cve", "advisory")), + ("msftsecintel", "security", ("cve", "advisory")), + ("Unit42_Intel", "security", ("intel", "cve")), + ("Mandiant", "security", ("intel", "cve")), + ("CrowdStrike", "security", ("intel", "cve")), + ("SentinelOne", "security", ("intel", "cve")), + ("Kaspersky", "security", ("intel", "cve")), + ("elasticlabs", "security", ("intel",)), + ("blackorbird", "security", ("apt", "intel")), + ("OpenAI", "models", ("model", "release")), + ("AnthropicAI", "models", ("model", "release")), + ("deepseek_ai", "models", ("model", "release")), + ("huggingface", "models", ("model", "dataset")), + ("AI_News_", "models", ("news", "model")), + ("karpathy", "community", ("ai", "guide")), + ("simonw", "community", ("ai", "tool")), + ("ylecun", "community", ("ai", "news")), +] + +X_SOURCES = [ + { + "name": "x_" + h.lower(), + "category": cat, + "kind": "tweet", + "url": "https://x.com/" + h, + "adapter": "x_user", + "tags": tags, + "limit": 15, + "extra": {"handle": h}, + } + for h, cat, tags in _X +] \ No newline at end of file