Performance: persistent event loop, batch DB, directory pre-scan
- Replace per-operation thread spawning with a single persistent asyncio event loop (saves ~10-50ms per async operation) - Pre-scan saved directories into sets instead of per-post exists() calls (~80+ syscalls reduced to a few iterdir()) - Add add_favorites_batch() for single-transaction bulk inserts - Add missing indexes on favorites.folder and favorites.favorited_at
This commit is contained in:
parent
f0afe52743
commit
afa08ff007
@ -41,6 +41,8 @@ CREATE TABLE IF NOT EXISTS favorites (
|
|||||||
|
|
||||||
CREATE INDEX IF NOT EXISTS idx_favorites_tags ON favorites(tags);
|
CREATE INDEX IF NOT EXISTS idx_favorites_tags ON favorites(tags);
|
||||||
CREATE INDEX IF NOT EXISTS idx_favorites_site ON favorites(site_id);
|
CREATE INDEX IF NOT EXISTS idx_favorites_site ON favorites(site_id);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_favorites_folder ON favorites(folder);
|
||||||
|
CREATE INDEX IF NOT EXISTS idx_favorites_favorited_at ON favorites(favorited_at DESC);
|
||||||
|
|
||||||
CREATE TABLE IF NOT EXISTS favorite_folders (
|
CREATE TABLE IF NOT EXISTS favorite_folders (
|
||||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||||
@ -247,6 +249,19 @@ class Database:
|
|||||||
favorited_at=now,
|
favorited_at=now,
|
||||||
)
|
)
|
||||||
|
|
||||||
|
def add_favorites_batch(self, favorites: list[dict]) -> None:
|
||||||
|
"""Add multiple favorites in a single transaction."""
|
||||||
|
for fav in favorites:
|
||||||
|
self.conn.execute(
|
||||||
|
"INSERT OR IGNORE INTO favorites "
|
||||||
|
"(site_id, post_id, file_url, preview_url, tags, rating, score, source, cached_path, folder, favorited_at) "
|
||||||
|
"VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||||
|
(fav['site_id'], fav['post_id'], fav['file_url'], fav.get('preview_url'),
|
||||||
|
fav.get('tags', ''), fav.get('rating'), fav.get('score'), fav.get('source'),
|
||||||
|
fav.get('cached_path'), fav.get('folder'), fav.get('favorited_at', datetime.now(timezone.utc).isoformat())),
|
||||||
|
)
|
||||||
|
self.conn.commit()
|
||||||
|
|
||||||
def remove_favorite(self, site_id: int, post_id: int) -> None:
|
def remove_favorite(self, site_id: int, post_id: int) -> None:
|
||||||
self.conn.execute(
|
self.conn.execute(
|
||||||
"DELETE FROM favorites WHERE site_id = ? AND post_id = ?",
|
"DELETE FROM favorites WHERE site_id = ? AND post_id = ?",
|
||||||
|
|||||||
@ -178,6 +178,10 @@ class BooruApp(QMainWindow):
|
|||||||
self._last_scroll_page = 0
|
self._last_scroll_page = 0
|
||||||
self._signals = AsyncSignals()
|
self._signals = AsyncSignals()
|
||||||
|
|
||||||
|
self._async_loop = asyncio.new_event_loop()
|
||||||
|
self._async_thread = threading.Thread(target=self._async_loop.run_forever, daemon=True)
|
||||||
|
self._async_thread.start()
|
||||||
|
|
||||||
self._setup_signals()
|
self._setup_signals()
|
||||||
self._setup_ui()
|
self._setup_ui()
|
||||||
self._setup_menu()
|
self._setup_menu()
|
||||||
@ -213,12 +217,7 @@ class BooruApp(QMainWindow):
|
|||||||
self._status.showMessage(f"Error: {e}")
|
self._status.showMessage(f"Error: {e}")
|
||||||
|
|
||||||
def _run_async(self, coro_func, *args):
|
def _run_async(self, coro_func, *args):
|
||||||
def _worker():
|
asyncio.run_coroutine_threadsafe(coro_func(*args), self._async_loop)
|
||||||
try:
|
|
||||||
asyncio.run(coro_func(*args))
|
|
||||||
except Exception as e:
|
|
||||||
log.error(f"Async worker failed: {e}")
|
|
||||||
threading.Thread(target=_worker, daemon=True).start()
|
|
||||||
|
|
||||||
def _setup_ui(self) -> None:
|
def _setup_ui(self) -> None:
|
||||||
central = QWidget()
|
central = QWidget()
|
||||||
@ -591,23 +590,31 @@ class BooruApp(QMainWindow):
|
|||||||
|
|
||||||
from ..core.config import saved_dir, saved_folder_dir
|
from ..core.config import saved_dir, saved_folder_dir
|
||||||
site_id = self._site_combo.currentData()
|
site_id = self._site_combo.currentData()
|
||||||
|
|
||||||
|
# Pre-scan saved directories once instead of per-post exists() calls
|
||||||
|
_sd = saved_dir()
|
||||||
|
_saved_ids: set[int] = set()
|
||||||
|
if _sd.exists():
|
||||||
|
_saved_ids = {int(f.stem) for f in _sd.iterdir() if f.is_file() and f.stem.isdigit()}
|
||||||
|
_folder_saved: dict[str, set[int]] = {}
|
||||||
|
for folder in self._db.get_folders():
|
||||||
|
d = saved_folder_dir(folder)
|
||||||
|
if d.exists():
|
||||||
|
_folder_saved[folder] = {int(f.stem) for f in d.iterdir() if f.is_file() and f.stem.isdigit()}
|
||||||
|
|
||||||
|
# Pre-fetch favorites for the site once (used for folder checks)
|
||||||
|
_favs = self._db.get_favorites(site_id=site_id) if site_id else []
|
||||||
|
|
||||||
for i, (post, thumb) in enumerate(zip(posts, thumbs)):
|
for i, (post, thumb) in enumerate(zip(posts, thumbs)):
|
||||||
if site_id and self._db.is_favorited(site_id, post.id):
|
if site_id and self._db.is_favorited(site_id, post.id):
|
||||||
thumb.set_favorited(True)
|
thumb.set_favorited(True)
|
||||||
# Check if saved to library (not just cached)
|
# Check if saved to library (not just cached)
|
||||||
saved = any(
|
saved = post.id in _saved_ids
|
||||||
(saved_dir() / f"{post.id}{ext}").exists()
|
|
||||||
for ext in MEDIA_EXTENSIONS
|
|
||||||
)
|
|
||||||
if not saved:
|
if not saved:
|
||||||
# Check folders
|
# Check folders
|
||||||
favs = self._db.get_favorites(site_id=site_id)
|
for f in _favs:
|
||||||
for f in favs:
|
if f.post_id == post.id and f.folder and f.folder in _folder_saved:
|
||||||
if f.post_id == post.id and f.folder:
|
saved = post.id in _folder_saved[f.folder]
|
||||||
saved = any(
|
|
||||||
(saved_folder_dir(f.folder) / f"{post.id}{ext}").exists()
|
|
||||||
for ext in MEDIA_EXTENSIONS
|
|
||||||
)
|
|
||||||
break
|
break
|
||||||
thumb.set_saved_locally(saved)
|
thumb.set_saved_locally(saved)
|
||||||
# Set drag path from cache
|
# Set drag path from cache
|
||||||
@ -1368,6 +1375,7 @@ class BooruApp(QMainWindow):
|
|||||||
thumbs[index].set_saved_locally(True)
|
thumbs[index].set_saved_locally(True)
|
||||||
|
|
||||||
def closeEvent(self, event) -> None:
|
def closeEvent(self, event) -> None:
|
||||||
|
self._async_loop.call_soon_threadsafe(self._async_loop.stop)
|
||||||
if self._db.get_setting_bool("clear_cache_on_exit"):
|
if self._db.get_setting_bool("clear_cache_on_exit"):
|
||||||
from ..core.cache import clear_cache
|
from ..core.cache import clear_cache
|
||||||
clear_cache(clear_images=True, clear_thumbnails=True)
|
clear_cache(clear_images=True, clear_thumbnails=True)
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user