31 lines
936 B
Python
31 lines
936 B
Python
#!/usr/bin/env python
|
|
# -*- coding: utf-8 -*-
|
|
"""Flatten a bank.jsonl into a proxy pool: one JSON line per proxy URL.
|
|
|
|
Usage: python collect.py [bank.jsonl] > pool.jsonl
|
|
"""
|
|
import json
|
|
import sys
|
|
|
|
src = sys.argv[1] if len(sys.argv) > 1 else "bank.jsonl"
|
|
seen = set()
|
|
out = sys.stdout
|
|
n = 0
|
|
for line in open(src, encoding="utf-8"):
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
rec = json.loads(line)
|
|
except Exception:
|
|
continue
|
|
for p in rec.get("proxies") or []:
|
|
url = p.get("url")
|
|
if not url or url in seen:
|
|
continue
|
|
seen.add(url)
|
|
row = {"acct": rec.get("idx"), "name": rec.get("name"), "cc": p.get("cc"),
|
|
"url": url, "socks": p.get("socks"), "client_id": p.get("client_id")}
|
|
out.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
n += 1
|
|
sys.stderr.write("proxies: %d (unique) from %d bank lines\n" % (n, len(seen))) |