http: consolidate httpx.AsyncClient construction into make_client
Three call sites built near-identical httpx.AsyncClient instances: the cache download pool, BooruClient's shared API pool, and detect_site_type's reach into that same pool. They differed only in timeout (60s vs 20s), Accept header (cache pool only), and which extra request hooks to attach. core/http.py:make_client is the single constructor now. Each call site still keeps its own singleton + lock (separate connection pools for large transfers vs short JSON), so this is a constructor consolidation, not a pool consolidation. No behavior change. Drops now-unused USER_AGENT imports from cache.py and base.py; make_client pulls it from core.config.
This commit is contained in:
parent
90b27fe36a
commit
ab44735f28
@ -16,6 +16,7 @@
|
|||||||
### Refactored
|
### Refactored
|
||||||
- `category_fetcher` batch tag-API params are now built by a shared `_build_tag_api_params` helper instead of duplicated across `fetch_via_tag_api` and `_probe_batch_api`
|
- `category_fetcher` batch tag-API params are now built by a shared `_build_tag_api_params` helper instead of duplicated across `fetch_via_tag_api` and `_probe_batch_api`
|
||||||
- `detect.detect_site_type` — removed the leftover `if True:` indent marker; no behavior change
|
- `detect.detect_site_type` — removed the leftover `if True:` indent marker; no behavior change
|
||||||
|
- `core.http.make_client` — single constructor for the three `httpx.AsyncClient` instances (cache download pool, API pool, detect probe). Each call site still keeps its own singleton and connection pool; only the construction is shared
|
||||||
|
|
||||||
## v0.2.7
|
## v0.2.7
|
||||||
|
|
||||||
|
|||||||
@ -10,9 +10,9 @@ from dataclasses import dataclass, field
|
|||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
|
|
||||||
from ..config import USER_AGENT, DEFAULT_PAGE_SIZE
|
from ..config import DEFAULT_PAGE_SIZE
|
||||||
from ..cache import log_connection
|
from ..cache import log_connection
|
||||||
from ._safety import redact_url, validate_public_request
|
from ._safety import redact_url
|
||||||
|
|
||||||
log = logging.getLogger("booru")
|
log = logging.getLogger("booru")
|
||||||
|
|
||||||
@ -100,21 +100,11 @@ class BooruClient(ABC):
|
|||||||
return c
|
return c
|
||||||
# Slow path: build it. Lock so two coroutines on the same loop don't
|
# Slow path: build it. Lock so two coroutines on the same loop don't
|
||||||
# both construct + leak.
|
# both construct + leak.
|
||||||
|
from ..http import make_client
|
||||||
with BooruClient._shared_client_lock:
|
with BooruClient._shared_client_lock:
|
||||||
c = BooruClient._shared_client
|
c = BooruClient._shared_client
|
||||||
if c is None or c.is_closed:
|
if c is None or c.is_closed:
|
||||||
c = httpx.AsyncClient(
|
c = make_client(extra_request_hooks=[self._log_request])
|
||||||
headers={"User-Agent": USER_AGENT},
|
|
||||||
follow_redirects=True,
|
|
||||||
timeout=20.0,
|
|
||||||
event_hooks={
|
|
||||||
"request": [
|
|
||||||
validate_public_request,
|
|
||||||
self._log_request,
|
|
||||||
],
|
|
||||||
},
|
|
||||||
limits=httpx.Limits(max_connections=10, max_keepalive_connections=5),
|
|
||||||
)
|
|
||||||
BooruClient._shared_client = c
|
BooruClient._shared_client = c
|
||||||
return c
|
return c
|
||||||
|
|
||||||
|
|||||||
@ -4,10 +4,7 @@ from __future__ import annotations
|
|||||||
|
|
||||||
import logging
|
import logging
|
||||||
|
|
||||||
import httpx
|
from ..http import make_client
|
||||||
|
|
||||||
from ..config import USER_AGENT
|
|
||||||
from ._safety import validate_public_request
|
|
||||||
from .danbooru import DanbooruClient
|
from .danbooru import DanbooruClient
|
||||||
from .gelbooru import GelbooruClient
|
from .gelbooru import GelbooruClient
|
||||||
from .moebooru import MoebooruClient
|
from .moebooru import MoebooruClient
|
||||||
@ -29,22 +26,11 @@ async def detect_site_type(
|
|||||||
url = url.rstrip("/")
|
url = url.rstrip("/")
|
||||||
|
|
||||||
from .base import BooruClient as _BC
|
from .base import BooruClient as _BC
|
||||||
# Reuse shared client for site detection. event_hooks mirrors
|
# Reuse shared client for site detection. Event hooks mirror
|
||||||
# BooruClient.client so detection requests get the same SSRF
|
# BooruClient.client so detection requests get the same SSRF
|
||||||
# validation and connection logging as regular API calls.
|
# validation and connection logging as regular API calls.
|
||||||
if _BC._shared_client is None or _BC._shared_client.is_closed:
|
if _BC._shared_client is None or _BC._shared_client.is_closed:
|
||||||
_BC._shared_client = httpx.AsyncClient(
|
_BC._shared_client = make_client(extra_request_hooks=[_BC._log_request])
|
||||||
headers={"User-Agent": USER_AGENT},
|
|
||||||
follow_redirects=True,
|
|
||||||
timeout=20.0,
|
|
||||||
event_hooks={
|
|
||||||
"request": [
|
|
||||||
validate_public_request,
|
|
||||||
_BC._log_request,
|
|
||||||
],
|
|
||||||
},
|
|
||||||
limits=httpx.Limits(max_connections=10, max_keepalive_connections=5),
|
|
||||||
)
|
|
||||||
client = _BC._shared_client
|
client = _BC._shared_client
|
||||||
# Try Danbooru / e621 first — /posts.json is a definitive endpoint
|
# Try Danbooru / e621 first — /posts.json is a definitive endpoint
|
||||||
try:
|
try:
|
||||||
|
|||||||
@ -17,7 +17,7 @@ from urllib.parse import urlparse
|
|||||||
import httpx
|
import httpx
|
||||||
from PIL import Image
|
from PIL import Image
|
||||||
|
|
||||||
from .config import cache_dir, thumbnails_dir, USER_AGENT
|
from .config import cache_dir, thumbnails_dir
|
||||||
|
|
||||||
log = logging.getLogger("booru")
|
log = logging.getLogger("booru")
|
||||||
|
|
||||||
@ -77,23 +77,14 @@ def _get_shared_client(referer: str = "") -> httpx.AsyncClient:
|
|||||||
c = _shared_client
|
c = _shared_client
|
||||||
if c is not None and not c.is_closed:
|
if c is not None and not c.is_closed:
|
||||||
return c
|
return c
|
||||||
# Lazy import: core.api.base imports log_connection from this
|
# Lazy import: core.http imports from core.api._safety, which
|
||||||
# module, so a top-level `from .api._safety import ...` would
|
# lives inside the api package that imports this module, so a
|
||||||
# circular-import through api/__init__.py during cache.py load.
|
# top-level import would circular through cache.py's load.
|
||||||
from .api._safety import validate_public_request
|
from .http import make_client
|
||||||
with _shared_client_lock:
|
with _shared_client_lock:
|
||||||
c = _shared_client
|
c = _shared_client
|
||||||
if c is None or c.is_closed:
|
if c is None or c.is_closed:
|
||||||
c = httpx.AsyncClient(
|
c = make_client(timeout=60.0, accept="image/*,video/*,*/*")
|
||||||
headers={
|
|
||||||
"User-Agent": USER_AGENT,
|
|
||||||
"Accept": "image/*,video/*,*/*",
|
|
||||||
},
|
|
||||||
follow_redirects=True,
|
|
||||||
timeout=60.0,
|
|
||||||
event_hooks={"request": [validate_public_request]},
|
|
||||||
limits=httpx.Limits(max_connections=10, max_keepalive_connections=5),
|
|
||||||
)
|
|
||||||
_shared_client = c
|
_shared_client = c
|
||||||
return c
|
return c
|
||||||
|
|
||||||
|
|||||||
73
booru_viewer/core/http.py
Normal file
73
booru_viewer/core/http.py
Normal file
@ -0,0 +1,73 @@
|
|||||||
|
"""Shared httpx.AsyncClient constructor.
|
||||||
|
|
||||||
|
Three call sites build near-identical clients: the cache module's
|
||||||
|
download pool, ``BooruClient``'s shared API pool, and
|
||||||
|
``detect.detect_site_type``'s reach into that same pool. Centralising
|
||||||
|
the construction in one place means a future change (new SSRF hook,
|
||||||
|
new connection limit, different default UA) doesn't have to be made
|
||||||
|
three times and kept in sync.
|
||||||
|
|
||||||
|
The module does NOT manage the singletons themselves — each call site
|
||||||
|
keeps its own ``_shared_client`` and its own lock, so the cache
|
||||||
|
pool's long-lived large transfers don't compete with short JSON
|
||||||
|
requests from the API layer. ``make_client`` is a pure constructor.
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
from typing import Callable, Iterable
|
||||||
|
|
||||||
|
import httpx
|
||||||
|
|
||||||
|
from .config import USER_AGENT
|
||||||
|
from .api._safety import validate_public_request
|
||||||
|
|
||||||
|
|
||||||
|
# Connection pool limits are identical across all three call sites.
|
||||||
|
# Keeping the default here centralises any future tuning.
|
||||||
|
_DEFAULT_LIMITS = httpx.Limits(max_connections=10, max_keepalive_connections=5)
|
||||||
|
|
||||||
|
|
||||||
|
def make_client(
|
||||||
|
*,
|
||||||
|
timeout: float = 20.0,
|
||||||
|
accept: str | None = None,
|
||||||
|
extra_request_hooks: Iterable[Callable] | None = None,
|
||||||
|
) -> httpx.AsyncClient:
|
||||||
|
"""Return a fresh ``httpx.AsyncClient`` with the project's defaults.
|
||||||
|
|
||||||
|
Defaults applied unconditionally:
|
||||||
|
- ``User-Agent`` header from ``core.config.USER_AGENT``
|
||||||
|
- ``follow_redirects=True``
|
||||||
|
- ``validate_public_request`` SSRF hook (always first on the
|
||||||
|
request-hook chain; extras run after it)
|
||||||
|
- Connection limits: 10 max, 5 keepalive
|
||||||
|
|
||||||
|
Parameters:
|
||||||
|
timeout: per-request timeout in seconds. Cache downloads pass
|
||||||
|
60s for large videos; the API pool uses 20s.
|
||||||
|
accept: optional ``Accept`` header value. The cache pool sets
|
||||||
|
``image/*,video/*,*/*``; the API pool leaves it unset so
|
||||||
|
httpx's ``*/*`` default takes effect.
|
||||||
|
extra_request_hooks: optional extra callables to run after
|
||||||
|
``validate_public_request``. The API clients pass their
|
||||||
|
connection-logging hook here; detect passes the same.
|
||||||
|
|
||||||
|
Call sites are responsible for their own singleton caching —
|
||||||
|
``make_client`` always returns a fresh instance.
|
||||||
|
"""
|
||||||
|
headers: dict[str, str] = {"User-Agent": USER_AGENT}
|
||||||
|
if accept is not None:
|
||||||
|
headers["Accept"] = accept
|
||||||
|
|
||||||
|
hooks: list[Callable] = [validate_public_request]
|
||||||
|
if extra_request_hooks:
|
||||||
|
hooks.extend(extra_request_hooks)
|
||||||
|
|
||||||
|
return httpx.AsyncClient(
|
||||||
|
headers=headers,
|
||||||
|
follow_redirects=True,
|
||||||
|
timeout=timeout,
|
||||||
|
event_hooks={"request": hooks},
|
||||||
|
limits=_DEFAULT_LIMITS,
|
||||||
|
)
|
||||||
Loading…
x
Reference in New Issue
Block a user