B

app/services/fetch_cache.py

@@ -0,0 +1,52 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import time
+from pathlib import Path
+
+from pydantic import BaseModel, Field
+
+
+class FetchCacheEntry(BaseModel):
+    text: str
+    headers: dict[str, str] = Field(default_factory=dict)
+
+
+def build_fetch_cache_key(*, name: str, url: str) -> str:
+    raw = f"{name}|{url}"
+    return hashlib.sha256(raw.encode("utf-8")).hexdigest()
+
+
+def load_fetch_cache(*, cache_dir: Path, cache_key: str, ttl_seconds: int) -> FetchCacheEntry | None:
+    text_path = cache_dir / f"{cache_key}.txt"
+    meta_path = cache_dir / f"{cache_key}.json"
+    if not text_path.is_file() or not meta_path.is_file():
+        return None
+
+    expires_at = meta_path.stat().st_mtime + ttl_seconds
+    if expires_at < time.time():
+        return None
+
+    try:
+        metadata = json.loads(meta_path.read_text(encoding="utf-8"))
+    except json.JSONDecodeError:
+        return None
+
+    headers = metadata.get("headers")
+    if not isinstance(headers, dict):
+        headers = {}
+
+    return FetchCacheEntry(
+        text=text_path.read_text(encoding="utf-8"),
+        headers={str(k): str(v) for k, v in headers.items()},
+    )
+
+
+def save_fetch_cache(*, cache_dir: Path, cache_key: str, text: str, headers: dict[str, str]) -> Path:
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    text_path = cache_dir / f"{cache_key}.txt"
+    meta_path = cache_dir / f"{cache_key}.json"
+    text_path.write_text(text, encoding="utf-8")
+    meta_path.write_text(json.dumps({"headers": headers}, ensure_ascii=False, indent=2), encoding="utf-8")
+    return text_path

app/services/subscriptions.py

@@ -12,6 +12,7 @@ import yaml
 from app.config import get_settings
 from app.models import FetchResult, ProviderDocument, SourceConfig, SourceSnapshot
 from app.services.cache import TTLCache
+from app.services.fetch_cache import build_fetch_cache_key, load_fetch_cache, save_fetch_cache
 from app.services.headers import parse_subscription_userinfo
 
 
@@ -27,6 +28,17 @@ async def fetch_source(name: str, source: SourceConfig) -> FetchResult:
     if cached is not None:
         return cached
 
+    cache_key = build_fetch_cache_key(name=name, url=source.url)
+    disk_cached = load_fetch_cache(
+        cache_dir=settings.fetch_cache_dir,
+        cache_key=cache_key,
+        ttl_seconds=ttl,
+    )
+    if disk_cached is not None:
+        result = FetchResult(text=disk_cached.text, headers=disk_cached.headers)
+        _fetch_cache.set(name, result, ttl)
+        return result
+
     headers = {"User-Agent": settings.default_user_agent}
     headers.update(source.headers)
 
@@ -35,6 +47,12 @@ async def fetch_source(name: str, source: SourceConfig) -> FetchResult:
         response.raise_for_status()
         result = FetchResult(text=response.text, headers=dict(response.headers))
         _fetch_cache.set(name, result, ttl)
+        save_fetch_cache(
+            cache_dir=settings.fetch_cache_dir,
+            cache_key=cache_key,
+            text=result.text,
+            headers=result.headers,
+        )
         return result
 
 
@@ -154,11 +172,11 @@ def parse_source_proxies(text: str, source_kind: str) -> list[dict[str, Any]]:
 
 
 def parse_base64_uri_proxies(text: str) -> list[dict[str, Any]]:
-    decoded = decode_base64_subscription(text)
-    return parse_uri_text_proxies(decoded)
+    return parse_uri_text_proxies(decode_base64_subscription(text))
 
 
 def parse_uri_text_proxies(text: str) -> list[dict[str, Any]]:
+    text = normalize_uri_subscription_text(text)
     candidates = [
         line.strip()
         for line in text.splitlines()
@@ -208,6 +226,41 @@ def decode_base64_subscription(text: str) -> str:
         raise ValueError("Upstream content is not valid base64 subscription text") from exc
 
 
+def normalize_uri_subscription_text(text: str) -> str:
+    normalized = text.strip()
+    if not normalized:
+        return normalized
+
+    for _ in range(3):
+        base64_decoded = try_decode_base64_subscription(normalized)
+        if base64_decoded is not None and _looks_like_uri_payload(base64_decoded):
+            normalized = base64_decoded.strip()
+            continue
+
+        unquoted = unquote(normalized)
+        if unquoted != normalized and _looks_like_uri_payload(unquoted):
+            normalized = unquoted.strip()
+            continue
+
+        break
+
+    return normalized
+
+
+def try_decode_base64_subscription(text: str) -> str | None:
+    try:
+        return decode_base64_subscription(text)
+    except ValueError:
+        return None
+
+
+def _looks_like_uri_payload(text: str) -> bool:
+    compact = text.strip()
+    if not compact:
+        return False
+    return "://" in compact or "%3A%2F%2F" in compact.lower()
+
+
 def parse_anytls_uri(uri: str) -> dict[str, Any]:
     parsed = urlparse(uri)
     server = parsed.hostname