B

app/services/subscriptions.py

@@ -12,6 +12,7 @@ import yaml
 from app.config import get_settings
 from app.models import FetchResult, ProviderDocument, SourceConfig, SourceSnapshot
 from app.services.cache import TTLCache
+from app.services.fetch_cache import build_fetch_cache_key, load_fetch_cache, save_fetch_cache
 from app.services.headers import parse_subscription_userinfo
 
 
@@ -27,6 +28,17 @@ async def fetch_source(name: str, source: SourceConfig) -> FetchResult:
     if cached is not None:
         return cached
 
+    cache_key = build_fetch_cache_key(name=name, url=source.url)
+    disk_cached = load_fetch_cache(
+        cache_dir=settings.fetch_cache_dir,
+        cache_key=cache_key,
+        ttl_seconds=ttl,
+    )
+    if disk_cached is not None:
+        result = FetchResult(text=disk_cached.text, headers=disk_cached.headers)
+        _fetch_cache.set(name, result, ttl)
+        return result
+
     headers = {"User-Agent": settings.default_user_agent}
     headers.update(source.headers)
 
@@ -35,6 +47,12 @@ async def fetch_source(name: str, source: SourceConfig) -> FetchResult:
         response.raise_for_status()
         result = FetchResult(text=response.text, headers=dict(response.headers))
         _fetch_cache.set(name, result, ttl)
+        save_fetch_cache(
+            cache_dir=settings.fetch_cache_dir,
+            cache_key=cache_key,
+            text=result.text,
+            headers=result.headers,
+        )
         return result
 
 
@@ -154,11 +172,11 @@ def parse_source_proxies(text: str, source_kind: str) -> list[dict[str, Any]]:
 
 
 def parse_base64_uri_proxies(text: str) -> list[dict[str, Any]]:
-    decoded = decode_base64_subscription(text)
-    return parse_uri_text_proxies(decoded)
+    return parse_uri_text_proxies(decode_base64_subscription(text))
 
 
 def parse_uri_text_proxies(text: str) -> list[dict[str, Any]]:
+    text = normalize_uri_subscription_text(text)
     candidates = [
         line.strip()
         for line in text.splitlines()
@@ -208,6 +226,41 @@ def decode_base64_subscription(text: str) -> str:
         raise ValueError("Upstream content is not valid base64 subscription text") from exc
 
 
+def normalize_uri_subscription_text(text: str) -> str:
+    normalized = text.strip()
+    if not normalized:
+        return normalized
+
+    for _ in range(3):
+        base64_decoded = try_decode_base64_subscription(normalized)
+        if base64_decoded is not None and _looks_like_uri_payload(base64_decoded):
+            normalized = base64_decoded.strip()
+            continue
+
+        unquoted = unquote(normalized)
+        if unquoted != normalized and _looks_like_uri_payload(unquoted):
+            normalized = unquoted.strip()
+            continue
+
+        break
+
+    return normalized
+
+
+def try_decode_base64_subscription(text: str) -> str | None:
+    try:
+        return decode_base64_subscription(text)
+    except ValueError:
+        return None
+
+
+def _looks_like_uri_payload(text: str) -> bool:
+    compact = text.strip()
+    if not compact:
+        return False
+    return "://" in compact or "%3A%2F%2F" in compact.lower()
+
+
 def parse_anytls_uri(uri: str) -> dict[str, Any]:
     parsed = urlparse(uri)
     server = parsed.hostname