From 0dbb84d308d9e9ab61e17730cf7d65eac1bdefce Mon Sep 17 00:00:00 2001 From: riglen Date: Thu, 9 Apr 2026 11:05:06 +0800 Subject: [PATCH] B --- README.md | 3 ++ app/config.py | 1 + app/services/fetch_cache.py | 52 ++++++++++++++++++++++++++++++++ app/services/subscriptions.py | 57 +++++++++++++++++++++++++++++++++-- config/sources.yaml | 2 +- 5 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 app/services/fetch_cache.py diff --git a/README.md b/README.md index 053ca85..653fa63 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,8 @@ > 2. base64 编码的 URI 订阅 > 3. 明文 URI 订阅 > +> 对 URI 订阅还额外兼容了常见变种：外层 base64 解开后仍是 base64、以及整段 URI 文本再次经过 URL encode。 +> > 当前已兼容常见的 `anytls://`、`vless://`、`trojan://`、`ss://`、`vmess://`。 --- @@ -145,6 +147,7 @@ HEAD //bundle/stash.yaml?sources=airport-a,airport-b - 响应头同样只取第一个源的 `Subscription-Userinfo` - 生成后的完整 YAML 会缓存到 `output/bundle-cache/`，默认 600 秒内直接返回缓存 - 可用 `force_refresh=true` 强制跳过缓存并覆盖旧缓存文件 +- 上游订阅原始内容也会按 TTL 落盘缓存到 `data/fetch-cache/`，默认 900 秒 --- diff --git a/app/config.py b/app/config.py index dd7182b..5ef33b6 100644 --- a/app/config.py +++ b/app/config.py @@ -30,6 +30,7 @@ class Settings(BaseSettings): sources_file: Path = CONFIG_DIR / "sources.yaml" rules_dir: Path = CONFIG_DIR / "rules" bundle_cache_dir: Path = ROOT_DIR / "output" / "bundle-cache" + fetch_cache_dir: Path = DATA_DIR / "fetch-cache" model_config = SettingsConfigDict( env_file=ROOT_DIR / ".env", diff --git a/app/services/fetch_cache.py b/app/services/fetch_cache.py new file mode 100644 index 0000000..f5f7efa --- /dev/null +++ b/app/services/fetch_cache.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +import hashlib +import json +import time +from pathlib import Path + +from pydantic import BaseModel, Field + + +class FetchCacheEntry(BaseModel): + text: str + headers: dict[str, str] = Field(default_factory=dict) + + +def build_fetch_cache_key(*, name: str, url: str) -> str: + raw = f"{name}|{url}" + return hashlib.sha256(raw.encode("utf-8")).hexdigest() + + +def load_fetch_cache(*, cache_dir: Path, cache_key: str, ttl_seconds: int) -> FetchCacheEntry | None: + text_path = cache_dir / f"{cache_key}.txt" + meta_path = cache_dir / f"{cache_key}.json" + if not text_path.is_file() or not meta_path.is_file(): + return None + + expires_at = meta_path.stat().st_mtime + ttl_seconds + if expires_at < time.time(): + return None + + try: + metadata = json.loads(meta_path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return None + + headers = metadata.get("headers") + if not isinstance(headers, dict): + headers = {} + + return FetchCacheEntry( + text=text_path.read_text(encoding="utf-8"), + headers={str(k): str(v) for k, v in headers.items()}, + ) + + +def save_fetch_cache(*, cache_dir: Path, cache_key: str, text: str, headers: dict[str, str]) -> Path: + cache_dir.mkdir(parents=True, exist_ok=True) + text_path = cache_dir / f"{cache_key}.txt" + meta_path = cache_dir / f"{cache_key}.json" + text_path.write_text(text, encoding="utf-8") + meta_path.write_text(json.dumps({"headers": headers}, ensure_ascii=False, indent=2), encoding="utf-8") + return text_path diff --git a/app/services/subscriptions.py b/app/services/subscriptions.py index fbc6430..0c822be 100644 --- a/app/services/subscriptions.py +++ b/app/services/subscriptions.py @@ -12,6 +12,7 @@ import yaml from app.config import get_settings from app.models import FetchResult, ProviderDocument, SourceConfig, SourceSnapshot from app.services.cache import TTLCache +from app.services.fetch_cache import build_fetch_cache_key, load_fetch_cache, save_fetch_cache from app.services.headers import parse_subscription_userinfo @@ -27,6 +28,17 @@ async def fetch_source(name: str, source: SourceConfig) -> FetchResult: if cached is not None: return cached + cache_key = build_fetch_cache_key(name=name, url=source.url) + disk_cached = load_fetch_cache( + cache_dir=settings.fetch_cache_dir, + cache_key=cache_key, + ttl_seconds=ttl, + ) + if disk_cached is not None: + result = FetchResult(text=disk_cached.text, headers=disk_cached.headers) + _fetch_cache.set(name, result, ttl) + return result + headers = {"User-Agent": settings.default_user_agent} headers.update(source.headers) @@ -35,6 +47,12 @@ async def fetch_source(name: str, source: SourceConfig) -> FetchResult: response.raise_for_status() result = FetchResult(text=response.text, headers=dict(response.headers)) _fetch_cache.set(name, result, ttl) + save_fetch_cache( + cache_dir=settings.fetch_cache_dir, + cache_key=cache_key, + text=result.text, + headers=result.headers, + ) return result @@ -154,11 +172,11 @@ def parse_source_proxies(text: str, source_kind: str) -> list[dict[str, Any]]: def parse_base64_uri_proxies(text: str) -> list[dict[str, Any]]: - decoded = decode_base64_subscription(text) - return parse_uri_text_proxies(decoded) + return parse_uri_text_proxies(decode_base64_subscription(text)) def parse_uri_text_proxies(text: str) -> list[dict[str, Any]]: + text = normalize_uri_subscription_text(text) candidates = [ line.strip() for line in text.splitlines() @@ -208,6 +226,41 @@ def decode_base64_subscription(text: str) -> str: raise ValueError("Upstream content is not valid base64 subscription text") from exc +def normalize_uri_subscription_text(text: str) -> str: + normalized = text.strip() + if not normalized: + return normalized + + for _ in range(3): + base64_decoded = try_decode_base64_subscription(normalized) + if base64_decoded is not None and _looks_like_uri_payload(base64_decoded): + normalized = base64_decoded.strip() + continue + + unquoted = unquote(normalized) + if unquoted != normalized and _looks_like_uri_payload(unquoted): + normalized = unquoted.strip() + continue + + break + + return normalized + + +def try_decode_base64_subscription(text: str) -> str | None: + try: + return decode_base64_subscription(text) + except ValueError: + return None + + +def _looks_like_uri_payload(text: str) -> bool: + compact = text.strip() + if not compact: + return False + return "://" in compact or "%3A%2F%2F" in compact.lower() + + def parse_anytls_uri(uri: str) -> dict[str, Any]: parsed = urlparse(uri) server = parsed.hostname diff --git a/config/sources.yaml b/config/sources.yaml index b21d013..ca71295 100644 --- a/config/sources.yaml +++ b/config/sources.yaml @@ -11,7 +11,7 @@ sources: exclude_regex: "流量|重置|到期|续费|官网|离线|套餐" airport-b: - enabled: false + enabled: true display_name: B kind: auto url: ${AIRPORT_B_URL}