db

app/config.py

@@ -26,6 +26,8 @@ class Settings(BaseSettings):
     bundle_cache_ttl_seconds: int = 600
     max_proxy_name_length: int = 80
     default_user_agent: str = "sub-provider/0.2"
+    database_url: str = Field(default=f"sqlite:///{(DATA_DIR / 'app.db').resolve().as_posix()}")
+    database_echo: bool = False
 
     config_dir: Path = CONFIG_DIR
     sources_file: Path = CONFIG_DIR / "sources.yaml"

app/db.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+from functools import lru_cache
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from app.config import DATA_DIR, get_settings
+from app.db_models import Base
+
+
+def _sqlite_connect_args(database_url: str) -> dict[str, bool]:
+    if database_url.startswith("sqlite"):
+        return {"check_same_thread": False}
+    return {}
+
+
+@lru_cache(maxsize=1)
+def get_engine():
+    settings = get_settings()
+    DATA_DIR.mkdir(parents=True, exist_ok=True)
+    return create_engine(
+        settings.database_url,
+        echo=settings.database_echo,
+        future=True,
+        connect_args=_sqlite_connect_args(settings.database_url),
+    )
+
+
+@lru_cache(maxsize=1)
+def get_session_factory():
+    return sessionmaker(bind=get_engine(), autoflush=False, autocommit=False, future=True)
+
+
+def init_db() -> None:
+    Base.metadata.create_all(bind=get_engine())
+
+
+def get_session() -> Session:
+    return get_session_factory()()

app/db_models.py

@@ -0,0 +1,149 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text, UniqueConstraint
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+class TimestampMixin:
+    created_at: Mapped[datetime] = mapped_column(DateTime, default=datetime.utcnow, nullable=False)
+    updated_at: Mapped[datetime] = mapped_column(
+        DateTime,
+        default=datetime.utcnow,
+        onupdate=datetime.utcnow,
+        nullable=False,
+    )
+
+
+class AppSettingORM(Base, TimestampMixin):
+    __tablename__ = "app_settings"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    key: Mapped[str] = mapped_column(String(100), unique=True, nullable=False)
+    value: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class SourceORM(Base, TimestampMixin):
+    __tablename__ = "sources"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    key: Mapped[str] = mapped_column(String(100), unique=True, nullable=False)
+    enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+    kind: Mapped[str] = mapped_column(String(50), nullable=False)
+    url: Mapped[str] = mapped_column(Text, nullable=False)
+    display_name: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    headers_json: Mapped[str] = mapped_column(Text, default="{}", nullable=False)
+    include_regex: Mapped[str | None] = mapped_column(Text, nullable=True)
+    exclude_regex: Mapped[str | None] = mapped_column(Text, nullable=True)
+    prefix: Mapped[str] = mapped_column(String(255), default="", nullable=False)
+    suffix: Mapped[str] = mapped_column(String(255), default="", nullable=False)
+    cache_ttl_seconds: Mapped[int | None] = mapped_column(Integer, nullable=True)
+
+
+class ProfileORM(Base, TimestampMixin):
+    __tablename__ = "profiles"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    key: Mapped[str] = mapped_column(String(100), unique=True, nullable=False)
+    title: Mapped[str] = mapped_column(String(255), nullable=False)
+    provider_interval: Mapped[int] = mapped_column(Integer, default=21600, nullable=False)
+    rule_interval: Mapped[int] = mapped_column(Integer, default=86400, nullable=False)
+    test_url: Mapped[str] = mapped_column(Text, nullable=False)
+    test_interval: Mapped[int] = mapped_column(Integer, default=300, nullable=False)
+    main_policy: Mapped[str] = mapped_column(String(255), nullable=False)
+    source_policy: Mapped[str] = mapped_column(String(255), nullable=False)
+    mixed_auto_policy: Mapped[str] = mapped_column(String(255), nullable=False)
+    manual_policy: Mapped[str] = mapped_column(String(255), nullable=False)
+    direct_policy: Mapped[str] = mapped_column(String(255), nullable=False)
+    mode: Mapped[str] = mapped_column(String(50), default="rule", nullable=False)
+    allow_lan: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+    ipv6: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+    mixed_port: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    socks_port: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    log_level: Mapped[str | None] = mapped_column(String(50), nullable=True)
+
+    source_links: Mapped[list["ProfileSourceORM"]] = relationship(
+        back_populates="profile",
+        cascade="all, delete-orphan",
+    )
+    rule_links: Mapped[list["ProfileRuleModuleORM"]] = relationship(
+        back_populates="profile",
+        cascade="all, delete-orphan",
+    )
+    groups: Mapped[list["PolicyGroupORM"]] = relationship(
+        back_populates="profile",
+        cascade="all, delete-orphan",
+    )
+
+
+class ProfileSourceORM(Base):
+    __tablename__ = "profile_sources"
+    __table_args__ = (UniqueConstraint("profile_id", "source_id", name="uq_profile_source"),)
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    profile_id: Mapped[int] = mapped_column(ForeignKey("profiles.id"), nullable=False)
+    source_id: Mapped[int] = mapped_column(ForeignKey("sources.id"), nullable=False)
+    order_index: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+
+    profile: Mapped[ProfileORM] = relationship(back_populates="source_links")
+    source: Mapped[SourceORM] = relationship()
+
+
+class RuleModuleORM(Base, TimestampMixin):
+    __tablename__ = "rule_modules"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    key: Mapped[str] = mapped_column(String(100), unique=True, nullable=False)
+    file_path: Mapped[str | None] = mapped_column(Text, nullable=True)
+    behavior: Mapped[str] = mapped_column(String(50), nullable=False)
+    format: Mapped[str] = mapped_column(String(50), nullable=False)
+    policy: Mapped[str] = mapped_column(String(255), nullable=False)
+    no_resolve: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
+    payload_json: Mapped[str] = mapped_column(Text, default="[]", nullable=False)
+
+    profile_links: Mapped[list["ProfileRuleModuleORM"]] = relationship(
+        back_populates="rule_module",
+        cascade="all, delete-orphan",
+    )
+
+
+class ProfileRuleModuleORM(Base):
+    __tablename__ = "profile_rule_modules"
+    __table_args__ = (UniqueConstraint("profile_id", "rule_module_id", name="uq_profile_rule_module"),)
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    profile_id: Mapped[int] = mapped_column(ForeignKey("profiles.id"), nullable=False)
+    rule_module_id: Mapped[int] = mapped_column(ForeignKey("rule_modules.id"), nullable=False)
+    enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+    order_index: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    policy_override: Mapped[str | None] = mapped_column(String(255), nullable=True)
+    payload_override_json: Mapped[str | None] = mapped_column(Text, nullable=True)
+    no_resolve_override: Mapped[bool | None] = mapped_column(Boolean, nullable=True)
+
+    profile: Mapped[ProfileORM] = relationship(back_populates="rule_links")
+    rule_module: Mapped[RuleModuleORM] = relationship(back_populates="profile_links")
+
+
+class PolicyGroupORM(Base, TimestampMixin):
+    __tablename__ = "policy_groups"
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True)
+    profile_id: Mapped[int] = mapped_column(ForeignKey("profiles.id"), nullable=False)
+    group_kind: Mapped[str] = mapped_column(String(50), nullable=False)
+    name: Mapped[str] = mapped_column(String(255), nullable=False)
+    type: Mapped[str] = mapped_column(String(50), nullable=False)
+    order_index: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
+    proxies_json: Mapped[str | None] = mapped_column(Text, nullable=True)
+    filter_regex: Mapped[str | None] = mapped_column(Text, nullable=True)
+    tolerance: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    url: Mapped[str | None] = mapped_column(Text, nullable=True)
+    interval: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
+
+    profile: Mapped[ProfileORM] = relationship(back_populates="groups")

app/import_config.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import argparse
+
+from app.config import get_settings
+from app.services.config_store import import_yaml_config_to_db
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Import YAML config into the sub-provider database")
+    parser.add_argument(
+        "--replace",
+        action="store_true",
+        help="Clear existing config rows before import",
+    )
+    args = parser.parse_args()
+
+    settings = get_settings()
+    import_yaml_config_to_db(settings.sources_file, replace_existing=args.replace)
+    print(f"Imported config into {settings.database_url}")
+
+
+if __name__ == "__main__":
+    main()

app/services/config_store.py

@@ -0,0 +1,361 @@
+from __future__ import annotations
+
+import json
+
+from sqlalchemy import delete, select
+from sqlalchemy.orm import Session, selectinload
+
+from app.db import get_session, init_db
+from app.db_models import (
+    AppSettingORM,
+    PolicyGroupORM,
+    ProfileORM,
+    ProfileRuleModuleORM,
+    ProfileSourceORM,
+    RuleModuleORM,
+    SourceORM,
+)
+from app.models import AppConfig, ClientConfig, ProxyGroupConfig, RegionConfig, RuleConfig, SourceConfig
+from app.services.loader import load_yaml_app_config
+
+
+def _json_dumps(value) -> str:
+    return json.dumps(value, ensure_ascii=False)
+
+
+def _json_loads(value: str | None, default):
+    if not value:
+        return default
+    return json.loads(value)
+
+
+def import_yaml_config_to_db(config_path, *, replace_existing: bool = False) -> None:
+    app_config = load_yaml_app_config(config_path)
+    init_db()
+    with get_session() as session:
+        if replace_existing:
+            _clear_config_tables(session)
+
+        _upsert_setting(session, "public_path", app_config.public_path or "")
+
+        source_rows: list[SourceORM] = []
+        for key, source in app_config.sources.items():
+            row = _get_or_create_source(session, key)
+            row.enabled = source.enabled
+            row.kind = source.kind
+            row.url = source.url
+            row.display_name = source.display_name
+            row.headers_json = _json_dumps(source.headers)
+            row.include_regex = source.include_regex
+            row.exclude_regex = source.exclude_regex
+            row.prefix = source.prefix
+            row.suffix = source.suffix
+            row.cache_ttl_seconds = source.cache_ttl_seconds
+            source_rows.append(row)
+
+        module_rows: list[RuleModuleORM] = []
+        ordered_rule_keys = list(app_config.rules.keys())
+        for key, rule in app_config.rules.items():
+            row = _get_or_create_rule_module(session, key)
+            row.file_path = rule.file
+            row.behavior = rule.behavior
+            row.format = rule.format
+            row.policy = rule.policy
+            row.no_resolve = rule.no_resolve
+            row.payload_json = _json_dumps(rule.payload)
+            module_rows.append(row)
+
+        session.flush()
+
+        for client_key, client in app_config.clients.items():
+            profile = _get_or_create_profile(session, client_key)
+            _populate_profile(profile, client)
+            session.flush()
+
+            _replace_profile_sources(session, profile.id, source_rows)
+            _replace_profile_groups(session, profile.id, app_config)
+            _replace_profile_rule_links(session, profile.id, module_rows, ordered_rule_keys)
+
+        session.commit()
+
+
+def load_app_config_from_db() -> AppConfig | None:
+    init_db()
+    with get_session() as session:
+        profiles = list(
+            session.scalars(
+                select(ProfileORM)
+                .options(
+                    selectinload(ProfileORM.groups),
+                    selectinload(ProfileORM.rule_links).selectinload(ProfileRuleModuleORM.rule_module),
+                )
+                .order_by(ProfileORM.key)
+            )
+        )
+        sources = list(session.scalars(select(SourceORM).order_by(SourceORM.key)))
+        if not profiles or not sources:
+            return None
+
+        public_path = _setting_value(session, "public_path")
+        primary_profile = profiles[0]
+
+        regions: dict[str, RegionConfig] = {}
+        selector_groups: list[ProxyGroupConfig] = []
+        policy_groups: list[ProxyGroupConfig] = []
+
+        for group in sorted(primary_profile.groups, key=lambda item: (item.order_index, item.id)):
+            if not group.enabled:
+                continue
+            if group.group_kind == "region":
+                regions[_slugify(group.name)] = RegionConfig(
+                    name=group.name,
+                    filter=group.filter_regex or "",
+                    tolerance=group.tolerance or 50,
+                )
+                continue
+            target = selector_groups if group.group_kind == "selector" else policy_groups
+            target.append(
+                ProxyGroupConfig(
+                    name=group.name,
+                    type=group.type,
+                    proxies=_json_loads(group.proxies_json, []),
+                    filter=group.filter_regex,
+                    tolerance=group.tolerance,
+                    url=group.url,
+                    interval=group.interval,
+                )
+            )
+
+        rules: dict[str, RuleConfig] = {}
+        for link in sorted(primary_profile.rule_links, key=lambda item: (item.order_index, item.id)):
+            if not link.enabled:
+                continue
+            module = link.rule_module
+            rules[module.key] = RuleConfig(
+                file=module.file_path,
+                behavior=module.behavior,
+                format=module.format,
+                policy=link.policy_override or module.policy,
+                no_resolve=module.no_resolve if link.no_resolve_override is None else link.no_resolve_override,
+                payload=_json_loads(link.payload_override_json, _json_loads(module.payload_json, [])),
+            )
+
+        return AppConfig(
+            public_path=public_path or None,
+            sources={
+                source.key: SourceConfig(
+                    enabled=source.enabled,
+                    kind=source.kind,
+                    url=source.url,
+                    display_name=source.display_name,
+                    headers=_json_loads(source.headers_json, {}),
+                    include_regex=source.include_regex,
+                    exclude_regex=source.exclude_regex,
+                    prefix=source.prefix,
+                    suffix=source.suffix,
+                    cache_ttl_seconds=source.cache_ttl_seconds,
+                )
+                for source in sources
+            },
+            rules=rules,
+            clients={profile.key: _profile_to_client_config(profile) for profile in profiles},
+            regions=regions,
+            selector_groups=selector_groups,
+            policy_groups=policy_groups,
+        )
+
+
+def _clear_config_tables(session: Session) -> None:
+    session.execute(delete(ProfileRuleModuleORM))
+    session.execute(delete(PolicyGroupORM))
+    session.execute(delete(ProfileSourceORM))
+    session.execute(delete(ProfileORM))
+    session.execute(delete(RuleModuleORM))
+    session.execute(delete(SourceORM))
+    session.execute(delete(AppSettingORM))
+    session.flush()
+
+
+def _upsert_setting(session: Session, key: str, value: str) -> None:
+    row = session.scalar(select(AppSettingORM).where(AppSettingORM.key == key))
+    if row is None:
+        row = AppSettingORM(key=key, value=value)
+        session.add(row)
+    else:
+        row.value = value
+
+
+def _setting_value(session: Session, key: str) -> str | None:
+    row = session.scalar(select(AppSettingORM).where(AppSettingORM.key == key))
+    if row is None:
+        return None
+    return row.value
+
+
+def _get_or_create_source(session: Session, key: str) -> SourceORM:
+    row = session.scalar(select(SourceORM).where(SourceORM.key == key))
+    if row is None:
+        row = SourceORM(key=key, kind="auto", url="")
+        session.add(row)
+    return row
+
+
+def _get_or_create_rule_module(session: Session, key: str) -> RuleModuleORM:
+    row = session.scalar(select(RuleModuleORM).where(RuleModuleORM.key == key))
+    if row is None:
+        row = RuleModuleORM(key=key, behavior="classical", format="text", policy="DIRECT")
+        session.add(row)
+    return row
+
+
+def _get_or_create_profile(session: Session, key: str) -> ProfileORM:
+    row = session.scalar(select(ProfileORM).where(ProfileORM.key == key))
+    if row is None:
+        row = ProfileORM(
+            key=key,
+            title=key,
+            test_url="https://www.gstatic.com/generate_204",
+            main_policy="🚀 节点选择",
+            source_policy="☁️ 机场选择",
+            mixed_auto_policy="♻️ 自动选择",
+            manual_policy="🚀 手动切换",
+            direct_policy="DIRECT",
+        )
+        session.add(row)
+    return row
+
+
+def _populate_profile(profile: ProfileORM, client: ClientConfig) -> None:
+    profile.title = client.title
+    profile.provider_interval = client.provider_interval
+    profile.rule_interval = client.rule_interval
+    profile.test_url = str(client.test_url)
+    profile.test_interval = client.test_interval
+    profile.main_policy = client.main_policy
+    profile.source_policy = client.source_policy
+    profile.mixed_auto_policy = client.mixed_auto_policy
+    profile.manual_policy = client.manual_policy
+    profile.direct_policy = client.direct_policy
+    profile.mode = client.mode
+    profile.allow_lan = client.allow_lan
+    profile.ipv6 = client.ipv6
+    profile.mixed_port = client.mixed_port
+    profile.socks_port = client.socks_port
+    profile.log_level = client.log_level
+
+
+def _replace_profile_sources(session: Session, profile_id: int, source_rows: list[SourceORM]) -> None:
+    session.execute(delete(ProfileSourceORM).where(ProfileSourceORM.profile_id == profile_id))
+    for order_index, source in enumerate(source_rows):
+        session.add(
+            ProfileSourceORM(
+                profile_id=profile_id,
+                source_id=source.id,
+                order_index=order_index,
+                enabled=True,
+            )
+        )
+
+
+def _replace_profile_groups(session: Session, profile_id: int, app_config: AppConfig) -> None:
+    session.execute(delete(PolicyGroupORM).where(PolicyGroupORM.profile_id == profile_id))
+
+    order_index = 0
+    for region in app_config.regions.values():
+        session.add(
+            PolicyGroupORM(
+                profile_id=profile_id,
+                group_kind="region",
+                name=region.name,
+                type="url-test",
+                order_index=order_index,
+                filter_regex=region.filter,
+                tolerance=region.tolerance,
+                enabled=True,
+            )
+        )
+        order_index += 1
+
+    for group in app_config.selector_groups:
+        session.add(
+            PolicyGroupORM(
+                profile_id=profile_id,
+                group_kind="selector",
+                name=group.name,
+                type=group.type,
+                order_index=order_index,
+                proxies_json=_json_dumps(group.proxies),
+                filter_regex=group.filter,
+                tolerance=group.tolerance,
+                url=str(group.url) if group.url else None,
+                interval=group.interval,
+                enabled=True,
+            )
+        )
+        order_index += 1
+
+    for group in app_config.policy_groups:
+        session.add(
+            PolicyGroupORM(
+                profile_id=profile_id,
+                group_kind="policy",
+                name=group.name,
+                type=group.type,
+                order_index=order_index,
+                proxies_json=_json_dumps(group.proxies),
+                filter_regex=group.filter,
+                tolerance=group.tolerance,
+                url=str(group.url) if group.url else None,
+                interval=group.interval,
+                enabled=True,
+            )
+        )
+        order_index += 1
+
+
+def _replace_profile_rule_links(
+    session: Session,
+    profile_id: int,
+    module_rows: list[RuleModuleORM],
+    ordered_rule_keys: list[str],
+) -> None:
+    session.execute(delete(ProfileRuleModuleORM).where(ProfileRuleModuleORM.profile_id == profile_id))
+    module_by_key = {module.key: module for module in module_rows}
+    for order_index, key in enumerate(ordered_rule_keys):
+        module = module_by_key[key]
+        session.add(
+            ProfileRuleModuleORM(
+                profile_id=profile_id,
+                rule_module_id=module.id,
+                enabled=True,
+                order_index=order_index,
+            )
+        )
+
+
+def _profile_to_client_config(profile: ProfileORM) -> ClientConfig:
+    return ClientConfig(
+        title=profile.title,
+        provider_interval=profile.provider_interval,
+        rule_interval=profile.rule_interval,
+        test_url=profile.test_url,
+        test_interval=profile.test_interval,
+        main_policy=profile.main_policy,
+        source_policy=profile.source_policy,
+        mixed_auto_policy=profile.mixed_auto_policy,
+        manual_policy=profile.manual_policy,
+        direct_policy=profile.direct_policy,
+        mode=profile.mode,
+        allow_lan=profile.allow_lan,
+        ipv6=profile.ipv6,
+        mixed_port=profile.mixed_port,
+        socks_port=profile.socks_port,
+        log_level=profile.log_level,
+    )
+
+
+def _slugify(name: str) -> str:
+    text = "".join(char.lower() if char.isalnum() else "-" for char in name).strip("-")
+    while "--" in text:
+        text = text.replace("--", "-")
+    return text or "group"

app/services/loader.py

@@ -7,6 +7,7 @@ from typing import Any
 
 import yaml
 
+from app.db import init_db
 from app.models import AppConfig
 
 _ENV_PATTERN = re.compile(r"\$\{([A-Z0-9_]+)\}")
@@ -61,5 +62,15 @@ def _load_split_config(path: Path) -> dict[str, Any]:
 
 
 def load_app_config(path: Path) -> AppConfig:
+    init_db()
+    from app.services.config_store import load_app_config_from_db
+
+    db_config = load_app_config_from_db()
+    if db_config is not None:
+        return db_config
+    return load_yaml_app_config(path)
+
+
+def load_yaml_app_config(path: Path) -> AppConfig:
     expanded = _expand_env(_load_split_config(path))
     return AppConfig.model_validate(expanded)