94 lines
3.0 KiB
Python
Executable File
94 lines
3.0 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Link liveness sweep for Markdown files (stdlib only).
|
|
|
|
Usage:
|
|
python linkcheck.py <file.md> [<file2.md> ...] [--workers N] [--timeout S]
|
|
|
|
Checks every http(s) URL found in the markdown files.
|
|
- 403/405 are reported as BOTWALL (bot wall — site is alive, not dead)
|
|
- Other HTTP errors are reported with their code
|
|
- Network errors/timeouts are reported as DEAD
|
|
|
|
Output line format: CHECK <status> <url>
|
|
Exit code is always 0; the report is the output. Diagnostics go to stderr.
|
|
"""
|
|
import argparse
|
|
import concurrent.futures
|
|
import re
|
|
import ssl
|
|
import sys
|
|
import urllib.error
|
|
import urllib.request
|
|
|
|
URL_RE = re.compile(r"https?://[^\s()<>{}|\"'`]+")
|
|
UA = ("Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
|
"(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36")
|
|
|
|
|
|
def probe(url, method, timeout, ctx):
|
|
req = urllib.request.Request(url, headers={"User-Agent": UA}, method=method)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=timeout, context=ctx) as resp:
|
|
return resp.status
|
|
except urllib.error.HTTPError as e:
|
|
return e.code
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def check(url, timeout):
|
|
url = url.rstrip(".,;:!?)]}")
|
|
ctx = ssl.create_default_context()
|
|
ctx.check_hostname = False
|
|
ctx.verify_mode = ssl.CERT_NONE
|
|
for method in ("HEAD", "GET"):
|
|
status = probe(url, method, timeout, ctx)
|
|
if status is not None and status != 999:
|
|
if status in (403, 405):
|
|
return f"CHECK BOTWALL {status} {url}"
|
|
if status < 400:
|
|
return f"CHECK {status} {url}"
|
|
# 4xx/5xx from HEAD could be server refusing HEAD; try GET first
|
|
if method == "GET":
|
|
return f"CHECK {status} {url}"
|
|
continue
|
|
if method == "GET":
|
|
break
|
|
return f"CHECK DEAD {url}"
|
|
|
|
|
|
def collect_urls(files):
|
|
urls, seen = [], set()
|
|
for path in files:
|
|
try:
|
|
text = open(path, encoding="utf-8").read()
|
|
except OSError as e:
|
|
print(f"ERR cannot read {path}: {e}", file=sys.stderr)
|
|
continue
|
|
for u in URL_RE.findall(text):
|
|
if u not in seen:
|
|
seen.add(u)
|
|
urls.append(u)
|
|
return urls
|
|
|
|
|
|
def main():
|
|
ap = argparse.ArgumentParser(description="Link liveness sweep for Markdown files.")
|
|
ap.add_argument("files", nargs="+", metavar="file.md")
|
|
ap.add_argument("--workers", type=int, default=24, help="parallel workers (default: 24)")
|
|
ap.add_argument("--timeout", type=float, default=15.0, help="per-request timeout in s (default: 15)")
|
|
args = ap.parse_args()
|
|
|
|
urls = collect_urls(args.files)
|
|
if not urls:
|
|
print("CHECK NONE (no URLs found)")
|
|
return 0
|
|
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=args.workers) as pool:
|
|
for result in pool.map(lambda u: check(u, args.timeout), urls):
|
|
print(result)
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main()) |