Performance: persistent event loop, batch DB, directory pre-scan

- Replace per-operation thread spawning with a single persistent
  asyncio event loop (saves ~10-50ms per async operation)
- Pre-scan saved directories into sets instead of per-post
  exists() calls (~80+ syscalls reduced to a few iterdir())
- Add add_favorites_batch() for single-transaction bulk inserts
- Add missing indexes on favorites.folder and favorites.favorited_at
This commit is contained in:
pax 2026-04-04 20:19:22 -05:00
parent f0afe52743
commit afa08ff007
2 changed files with 40 additions and 17 deletions

View File

@ -41,6 +41,8 @@ CREATE TABLE IF NOT EXISTS favorites (
CREATE INDEX IF NOT EXISTS idx_favorites_tags ON favorites(tags);
CREATE INDEX IF NOT EXISTS idx_favorites_site ON favorites(site_id);
CREATE INDEX IF NOT EXISTS idx_favorites_folder ON favorites(folder);
CREATE INDEX IF NOT EXISTS idx_favorites_favorited_at ON favorites(favorited_at DESC);
CREATE TABLE IF NOT EXISTS favorite_folders (
id INTEGER PRIMARY KEY AUTOINCREMENT,
@ -247,6 +249,19 @@ class Database:
favorited_at=now,
)
def add_favorites_batch(self, favorites: list[dict]) -> None:
"""Add multiple favorites in a single transaction."""
for fav in favorites:
self.conn.execute(
"INSERT OR IGNORE INTO favorites "
"(site_id, post_id, file_url, preview_url, tags, rating, score, source, cached_path, folder, favorited_at) "
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(fav['site_id'], fav['post_id'], fav['file_url'], fav.get('preview_url'),
fav.get('tags', ''), fav.get('rating'), fav.get('score'), fav.get('source'),
fav.get('cached_path'), fav.get('folder'), fav.get('favorited_at', datetime.now(timezone.utc).isoformat())),
)
self.conn.commit()
def remove_favorite(self, site_id: int, post_id: int) -> None:
self.conn.execute(
"DELETE FROM favorites WHERE site_id = ? AND post_id = ?",

View File

@ -178,6 +178,10 @@ class BooruApp(QMainWindow):
self._last_scroll_page = 0
self._signals = AsyncSignals()
self._async_loop = asyncio.new_event_loop()
self._async_thread = threading.Thread(target=self._async_loop.run_forever, daemon=True)
self._async_thread.start()
self._setup_signals()
self._setup_ui()
self._setup_menu()
@ -213,12 +217,7 @@ class BooruApp(QMainWindow):
self._status.showMessage(f"Error: {e}")
def _run_async(self, coro_func, *args):
def _worker():
try:
asyncio.run(coro_func(*args))
except Exception as e:
log.error(f"Async worker failed: {e}")
threading.Thread(target=_worker, daemon=True).start()
asyncio.run_coroutine_threadsafe(coro_func(*args), self._async_loop)
def _setup_ui(self) -> None:
central = QWidget()
@ -591,23 +590,31 @@ class BooruApp(QMainWindow):
from ..core.config import saved_dir, saved_folder_dir
site_id = self._site_combo.currentData()
# Pre-scan saved directories once instead of per-post exists() calls
_sd = saved_dir()
_saved_ids: set[int] = set()
if _sd.exists():
_saved_ids = {int(f.stem) for f in _sd.iterdir() if f.is_file() and f.stem.isdigit()}
_folder_saved: dict[str, set[int]] = {}
for folder in self._db.get_folders():
d = saved_folder_dir(folder)
if d.exists():
_folder_saved[folder] = {int(f.stem) for f in d.iterdir() if f.is_file() and f.stem.isdigit()}
# Pre-fetch favorites for the site once (used for folder checks)
_favs = self._db.get_favorites(site_id=site_id) if site_id else []
for i, (post, thumb) in enumerate(zip(posts, thumbs)):
if site_id and self._db.is_favorited(site_id, post.id):
thumb.set_favorited(True)
# Check if saved to library (not just cached)
saved = any(
(saved_dir() / f"{post.id}{ext}").exists()
for ext in MEDIA_EXTENSIONS
)
saved = post.id in _saved_ids
if not saved:
# Check folders
favs = self._db.get_favorites(site_id=site_id)
for f in favs:
if f.post_id == post.id and f.folder:
saved = any(
(saved_folder_dir(f.folder) / f"{post.id}{ext}").exists()
for ext in MEDIA_EXTENSIONS
)
for f in _favs:
if f.post_id == post.id and f.folder and f.folder in _folder_saved:
saved = post.id in _folder_saved[f.folder]
break
thumb.set_saved_locally(saved)
# Set drag path from cache
@ -1368,6 +1375,7 @@ class BooruApp(QMainWindow):
thumbs[index].set_saved_locally(True)
def closeEvent(self, event) -> None:
self._async_loop.call_soon_threadsafe(self._async_loop.stop)
if self._db.get_setting_bool("clear_cache_on_exit"):
from ..core.cache import clear_cache
clear_cache(clear_images=True, clear_thumbnails=True)