step1
This commit is contained in:
riglen
69
app/conf_models.py
Normal file
69
app/conf_models.py
Normal file
@@ -0,0 +1,69 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class ConfSource(BaseModel):
|
||||
key: str
|
||||
source_type: Literal["url", "file", "inline", "base64"] = "url"
|
||||
value: str
|
||||
enabled: bool = True
|
||||
cache_ttl: int | None = None
|
||||
options: dict[str, str] = Field(default_factory=dict)
|
||||
line_no: int
|
||||
|
||||
|
||||
class ConfSelector(BaseModel):
|
||||
key: str
|
||||
regex: str
|
||||
line_no: int
|
||||
|
||||
|
||||
class ConfGroup(BaseModel):
|
||||
name: str
|
||||
group_type: str
|
||||
tokens: list[str] = Field(default_factory=list)
|
||||
raw: str
|
||||
line_no: int
|
||||
|
||||
|
||||
class ConfModule(BaseModel):
|
||||
key: str
|
||||
path: str
|
||||
policy: str
|
||||
order: int
|
||||
enabled: bool = True
|
||||
line_no: int
|
||||
|
||||
def resolved_path(self, config_dir: Path) -> Path:
|
||||
return (config_dir / self.path).resolve()
|
||||
|
||||
|
||||
class ConfBuiltin(BaseModel):
|
||||
key: str
|
||||
builtin_type: Literal["GEOIP", "FINAL"]
|
||||
value: str = ""
|
||||
policy: str
|
||||
order: int
|
||||
enabled: bool = True
|
||||
line_no: int
|
||||
|
||||
|
||||
class ConfBaseConfig(BaseModel):
|
||||
listen: str | None = None
|
||||
output_dir: str | None = None
|
||||
cache_dir: str | None = None
|
||||
mode: str | None = None
|
||||
allow_lan: bool | None = None
|
||||
log_level: str | None = None
|
||||
ipv6: bool | None = None
|
||||
append_userinfo_header: bool | None = None
|
||||
userinfo_source_policy: str | None = None
|
||||
sources: list[ConfSource] = Field(default_factory=list)
|
||||
selectors: list[ConfSelector] = Field(default_factory=list)
|
||||
groups: list[ConfGroup] = Field(default_factory=list)
|
||||
modules: list[ConfModule] = Field(default_factory=list)
|
||||
builtins: list[ConfBuiltin] = Field(default_factory=list)
|
||||
@@ -28,6 +28,7 @@ class Settings(BaseSettings):
|
||||
default_user_agent: str = "sub-provider/0.2"
|
||||
|
||||
sources_file: Path = CONFIG_DIR / "sources.yaml"
|
||||
conf_base_file: Path = CONFIG_DIR / "conf" / "base.conf"
|
||||
rules_dir: Path = CONFIG_DIR / "rules"
|
||||
bundle_cache_dir: Path = ROOT_DIR / "output" / "bundle-cache"
|
||||
fetch_cache_dir: Path = DATA_DIR / "fetch-cache"
|
||||
|
||||
222
app/services/conf_loader.py
Normal file
222
app/services/conf_loader.py
Normal file
@@ -0,0 +1,222 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import os
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from app.conf_models import ConfBaseConfig, ConfBuiltin, ConfGroup, ConfModule, ConfSelector, ConfSource
|
||||
|
||||
_ENV_PATTERN = re.compile(r"\$\{([A-Z0-9_]+)\}")
|
||||
_SCALAR_KEYS = {
|
||||
"listen",
|
||||
"output_dir",
|
||||
"cache_dir",
|
||||
"mode",
|
||||
"allow_lan",
|
||||
"log_level",
|
||||
"ipv6",
|
||||
"append_userinfo_header",
|
||||
"userinfo_source_policy",
|
||||
}
|
||||
_BOOL_KEYS = {"allow_lan", "ipv6", "append_userinfo_header"}
|
||||
_MULTI_KEYS = {"source", "selector", "group", "module", "builtin"}
|
||||
_SOURCE_TYPES = {"url", "file", "inline", "base64"}
|
||||
|
||||
|
||||
class ConfConfigError(ValueError):
|
||||
def __init__(self, path: Path, line_no: int, message: str) -> None:
|
||||
self.path = path
|
||||
self.line_no = line_no
|
||||
self.message = message
|
||||
super().__init__(f"{path}:{line_no}: {message}")
|
||||
|
||||
|
||||
def _expand_env(value: str) -> str:
|
||||
return _ENV_PATTERN.sub(lambda match: os.getenv(match.group(1), ""), value)
|
||||
|
||||
|
||||
def _parse_bool(value: str, *, path: Path, line_no: int, key: str) -> bool:
|
||||
normalized = value.strip().lower()
|
||||
mapping = {
|
||||
"true": True,
|
||||
"yes": True,
|
||||
"1": True,
|
||||
"false": False,
|
||||
"no": False,
|
||||
"0": False,
|
||||
}
|
||||
if normalized not in mapping:
|
||||
raise ConfConfigError(path, line_no, f"invalid boolean for {key}: {value}")
|
||||
return mapping[normalized]
|
||||
|
||||
|
||||
def _parse_csv_fields(value: str, *, path: Path, line_no: int) -> list[str]:
|
||||
try:
|
||||
row = next(csv.reader([value], skipinitialspace=True))
|
||||
except Exception as exc: # noqa: BLE001
|
||||
raise ConfConfigError(path, line_no, f"invalid csv payload: {exc}") from exc
|
||||
return [field.strip() for field in row]
|
||||
|
||||
|
||||
def _ensure_named_records(records: list[Any], *, key: str, path: Path) -> None:
|
||||
seen: dict[str, int] = {}
|
||||
for record in records:
|
||||
name = getattr(record, key)
|
||||
line_no = getattr(record, "line_no")
|
||||
previous_line = seen.get(name)
|
||||
if previous_line is not None:
|
||||
raise ConfConfigError(path, line_no, f"duplicate {record.__class__.__name__} key '{name}', first defined at line {previous_line}")
|
||||
seen[name] = line_no
|
||||
|
||||
|
||||
def _parse_source(value: str, *, path: Path, line_no: int) -> ConfSource:
|
||||
fields = _parse_csv_fields(_expand_env(value), path=path, line_no=line_no)
|
||||
if len(fields) < 3:
|
||||
raise ConfConfigError(path, line_no, "source requires at least 3 fields: key,type,value")
|
||||
key, source_type, source_value, *extras = fields
|
||||
if not key:
|
||||
raise ConfConfigError(path, line_no, "source key cannot be empty")
|
||||
if source_type not in _SOURCE_TYPES:
|
||||
raise ConfConfigError(path, line_no, f"unsupported source type: {source_type}")
|
||||
options: dict[str, str] = {}
|
||||
enabled = True
|
||||
cache_ttl: int | None = None
|
||||
for item in extras:
|
||||
if "=" not in item:
|
||||
raise ConfConfigError(path, line_no, f"source option must be k=v: {item}")
|
||||
option_key, option_value = [part.strip() for part in item.split("=", 1)]
|
||||
if not option_key:
|
||||
raise ConfConfigError(path, line_no, "source option key cannot be empty")
|
||||
option_value = _expand_env(option_value)
|
||||
if option_key == "enabled":
|
||||
enabled = _parse_bool(option_value, path=path, line_no=line_no, key="source.enabled")
|
||||
elif option_key == "cache_ttl":
|
||||
try:
|
||||
cache_ttl = int(option_value)
|
||||
except ValueError as exc:
|
||||
raise ConfConfigError(path, line_no, f"invalid integer for source.cache_ttl: {option_value}") from exc
|
||||
else:
|
||||
options[option_key] = option_value
|
||||
return ConfSource(
|
||||
key=key,
|
||||
source_type=source_type,
|
||||
value=source_value,
|
||||
enabled=enabled,
|
||||
cache_ttl=cache_ttl,
|
||||
options=options,
|
||||
line_no=line_no,
|
||||
)
|
||||
|
||||
|
||||
def _parse_selector(value: str, *, path: Path, line_no: int) -> ConfSelector:
|
||||
fields = _parse_csv_fields(value, path=path, line_no=line_no)
|
||||
if len(fields) != 2:
|
||||
raise ConfConfigError(path, line_no, "selector requires exactly 2 fields: key,regex")
|
||||
key, regex = fields
|
||||
if not key or not regex:
|
||||
raise ConfConfigError(path, line_no, "selector key and regex cannot be empty")
|
||||
return ConfSelector(key=key, regex=regex, line_no=line_no)
|
||||
|
||||
|
||||
def _parse_group(value: str, *, path: Path, line_no: int) -> ConfGroup:
|
||||
segments = [segment.strip() for segment in value.split("`")]
|
||||
if len(segments) < 2:
|
||||
raise ConfConfigError(path, line_no, "group requires at least name`type")
|
||||
name, group_type, *tokens = segments
|
||||
if not name or not group_type:
|
||||
raise ConfConfigError(path, line_no, "group name and type cannot be empty")
|
||||
return ConfGroup(name=name, group_type=group_type, tokens=tokens, raw=value.strip(), line_no=line_no)
|
||||
|
||||
|
||||
def _parse_module(value: str, *, path: Path, line_no: int, config_dir: Path) -> ConfModule:
|
||||
fields = _parse_csv_fields(value, path=path, line_no=line_no)
|
||||
if len(fields) != 5:
|
||||
raise ConfConfigError(path, line_no, "module requires exactly 5 fields: key,path,policy,order,enabled")
|
||||
key, module_path, policy, order_text, enabled_text = fields
|
||||
if not key or not module_path or not policy:
|
||||
raise ConfConfigError(path, line_no, "module key, path, and policy cannot be empty")
|
||||
try:
|
||||
order = int(order_text)
|
||||
except ValueError as exc:
|
||||
raise ConfConfigError(path, line_no, f"invalid integer for module.order: {order_text}") from exc
|
||||
enabled = _parse_bool(enabled_text, path=path, line_no=line_no, key="module.enabled")
|
||||
resolved_path = (config_dir / module_path).resolve()
|
||||
if not resolved_path.is_file():
|
||||
raise ConfConfigError(path, line_no, f"module path does not exist: {module_path}")
|
||||
return ConfModule(key=key, path=module_path, policy=policy, order=order, enabled=enabled, line_no=line_no)
|
||||
|
||||
|
||||
def _parse_builtin(value: str, *, path: Path, line_no: int) -> ConfBuiltin:
|
||||
fields = _parse_csv_fields(value, path=path, line_no=line_no)
|
||||
if len(fields) != 6:
|
||||
raise ConfConfigError(path, line_no, "builtin requires exactly 6 fields: key,type,value,policy,order,enabled")
|
||||
key, builtin_type, builtin_value, policy, order_text, enabled_text = fields
|
||||
if builtin_type not in {"GEOIP", "FINAL"}:
|
||||
raise ConfConfigError(path, line_no, f"unsupported builtin type: {builtin_type}")
|
||||
if not key or not policy:
|
||||
raise ConfConfigError(path, line_no, "builtin key and policy cannot be empty")
|
||||
if builtin_type == "GEOIP" and not builtin_value:
|
||||
raise ConfConfigError(path, line_no, "builtin GEOIP requires a value")
|
||||
try:
|
||||
order = int(order_text)
|
||||
except ValueError as exc:
|
||||
raise ConfConfigError(path, line_no, f"invalid integer for builtin.order: {order_text}") from exc
|
||||
enabled = _parse_bool(enabled_text, path=path, line_no=line_no, key="builtin.enabled")
|
||||
return ConfBuiltin(
|
||||
key=key,
|
||||
builtin_type=builtin_type,
|
||||
value=builtin_value,
|
||||
policy=policy,
|
||||
order=order,
|
||||
enabled=enabled,
|
||||
line_no=line_no,
|
||||
)
|
||||
|
||||
|
||||
def load_conf_base(path: Path) -> ConfBaseConfig:
|
||||
config = ConfBaseConfig()
|
||||
scalar_values: dict[str, Any] = {}
|
||||
config_dir = path.resolve().parent
|
||||
|
||||
for line_no, raw_line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
line = raw_line.strip()
|
||||
if not line or line.startswith("#") or line.startswith(";"):
|
||||
continue
|
||||
if "=" not in raw_line:
|
||||
raise ConfConfigError(path, line_no, "expected key = value")
|
||||
key, value = [part.strip() for part in raw_line.split("=", 1)]
|
||||
if not key:
|
||||
raise ConfConfigError(path, line_no, "key cannot be empty")
|
||||
if key in _SCALAR_KEYS:
|
||||
if key in scalar_values:
|
||||
raise ConfConfigError(path, line_no, f"duplicate scalar key: {key}")
|
||||
expanded_value = _expand_env(value)
|
||||
if key in _BOOL_KEYS:
|
||||
scalar_values[key] = _parse_bool(expanded_value, path=path, line_no=line_no, key=key)
|
||||
else:
|
||||
scalar_values[key] = expanded_value
|
||||
continue
|
||||
if key not in _MULTI_KEYS:
|
||||
raise ConfConfigError(path, line_no, f"unknown directive: {key}")
|
||||
if key == "source":
|
||||
config.sources.append(_parse_source(value, path=path, line_no=line_no))
|
||||
elif key == "selector":
|
||||
config.selectors.append(_parse_selector(value, path=path, line_no=line_no))
|
||||
elif key == "group":
|
||||
config.groups.append(_parse_group(value, path=path, line_no=line_no))
|
||||
elif key == "module":
|
||||
config.modules.append(_parse_module(value, path=path, line_no=line_no, config_dir=config_dir))
|
||||
elif key == "builtin":
|
||||
config.builtins.append(_parse_builtin(value, path=path, line_no=line_no))
|
||||
|
||||
for key, parsed_value in scalar_values.items():
|
||||
setattr(config, key, parsed_value)
|
||||
|
||||
_ensure_named_records(config.sources, key="key", path=path)
|
||||
_ensure_named_records(config.selectors, key="key", path=path)
|
||||
_ensure_named_records(config.groups, key="name", path=path)
|
||||
_ensure_named_records(config.modules, key="key", path=path)
|
||||
_ensure_named_records(config.builtins, key="key", path=path)
|
||||
return config
|
||||
Reference in New Issue
Block a user