Initial release — MediaTracker v1.0
Dockerized media library scanner for NAS drives. Identifies movies and series from filenames, fetches posters via the public IMDB suggestion API (no key required), and exposes a dark-themed web UI with multilingual search support. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
snoopy
18
backend/Dockerfile
Normal file
18
backend/Dockerfile
Normal file
@@ -0,0 +1,18 @@
|
||||
FROM python:3.12-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
gcc \
|
||||
default-libmysqlclient-dev \
|
||||
pkg-config \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
RUN mkdir -p /nas
|
||||
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000", "--log-level", "info"]
|
||||
71
backend/config.py
Normal file
71
backend/config.py
Normal file
@@ -0,0 +1,71 @@
|
||||
import os
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class NasConfig:
|
||||
index: int
|
||||
name: str
|
||||
container_path: str
|
||||
|
||||
|
||||
@dataclass
|
||||
class Settings:
|
||||
db_host: str
|
||||
db_port: int
|
||||
db_name: str
|
||||
db_user: str
|
||||
db_password: str
|
||||
scan_interval_hours: int
|
||||
video_extensions: frozenset[str]
|
||||
nas_list: list[NasConfig]
|
||||
|
||||
@property
|
||||
def db_url(self) -> str:
|
||||
return (
|
||||
f"mysql://{self.db_user}:{self.db_password}"
|
||||
f"@{self.db_host}:{self.db_port}/{self.db_name}"
|
||||
"?charset=utf8mb4"
|
||||
)
|
||||
|
||||
|
||||
def _discover_nas() -> list[NasConfig]:
|
||||
"""Detect mounted NAS volumes inside the container (/nas/nas1, /nas/nas2, …)."""
|
||||
nas_base = "/nas"
|
||||
entries: list[NasConfig] = []
|
||||
idx = 1
|
||||
while True:
|
||||
container_path = f"{nas_base}/nas{idx}"
|
||||
if not os.path.isdir(container_path):
|
||||
break
|
||||
# Attempt to read the name from env (set by generate-compose.py indirectly via NAS_X)
|
||||
env_key = f"NAS_{idx}"
|
||||
raw = os.getenv(env_key, "")
|
||||
if raw and "|" in raw:
|
||||
name = raw.split("|", 1)[0].strip()
|
||||
else:
|
||||
name = f"NAS {idx}"
|
||||
entries.append(NasConfig(index=idx, name=name, container_path=container_path))
|
||||
idx += 1
|
||||
if not entries:
|
||||
logger.warning("No NAS volumes found at %s/nasX — scanner will have nothing to scan", nas_base)
|
||||
return entries
|
||||
|
||||
|
||||
def load_settings() -> Settings:
|
||||
raw_ext = os.getenv("VIDEO_EXTENSIONS", ".mkv,.mp4,.avi,.m4v,.mov,.wmv,.ts,.m2ts,.iso")
|
||||
extensions = frozenset(e.strip().lower() for e in raw_ext.split(",") if e.strip())
|
||||
|
||||
return Settings(
|
||||
db_host=os.getenv("DB_HOST", "db"),
|
||||
db_port=int(os.getenv("DB_PORT", "3306")),
|
||||
db_name=os.getenv("DB_NAME", "mediatracker"),
|
||||
db_user=os.getenv("DB_USER", "mediatracker"),
|
||||
db_password=os.getenv("DB_PASSWORD", "changeme_db"),
|
||||
scan_interval_hours=int(os.getenv("SCAN_INTERVAL_HOURS", "6")),
|
||||
video_extensions=extensions,
|
||||
nas_list=_discover_nas(),
|
||||
)
|
||||
194
backend/database.py
Normal file
194
backend/database.py
Normal file
@@ -0,0 +1,194 @@
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
|
||||
from sqlalchemy import (
|
||||
create_engine,
|
||||
text,
|
||||
Column, Integer, SmallInteger, BigInteger, String, Text,
|
||||
DateTime, Enum, ForeignKey, UniqueConstraint, Index, and_,
|
||||
)
|
||||
from sqlalchemy.orm import (
|
||||
DeclarativeBase,
|
||||
relationship,
|
||||
Session,
|
||||
sessionmaker,
|
||||
)
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
# ─────────────────────────────────────────
|
||||
# ORM models
|
||||
# ─────────────────────────────────────────
|
||||
|
||||
class NasSource(Base):
|
||||
__tablename__ = "nas_sources"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
name = Column(String(255), nullable=False)
|
||||
path = Column(String(1024), nullable=False)
|
||||
last_scan = Column(DateTime, nullable=True)
|
||||
|
||||
movies = relationship("Movie", back_populates="nas", cascade="all, delete-orphan")
|
||||
seasons = relationship("Season", back_populates="nas", cascade="all, delete-orphan")
|
||||
unmatched = relationship("UnmatchedFile", back_populates="nas", cascade="all, delete-orphan")
|
||||
|
||||
|
||||
class Movie(Base):
|
||||
__tablename__ = "movies"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("file_path", name="uq_movie_path"),
|
||||
Index("idx_movie_title", "title"),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
title = Column(String(512), nullable=False)
|
||||
year = Column(SmallInteger, nullable=True)
|
||||
quality = Column(String(64), nullable=True)
|
||||
codec = Column(String(64), nullable=True)
|
||||
file_path = Column(String(2048), nullable=False)
|
||||
file_size = Column(BigInteger, nullable=True)
|
||||
poster_url = Column(String(512), nullable=True)
|
||||
imdb_id = Column(String(20), nullable=True)
|
||||
nas_id = Column(Integer, ForeignKey("nas_sources.id", ondelete="CASCADE"), nullable=False)
|
||||
added_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
|
||||
|
||||
nas = relationship("NasSource", back_populates="movies")
|
||||
|
||||
|
||||
class Series(Base):
|
||||
__tablename__ = "series"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("title_normalized", name="uq_series_normalized"),
|
||||
Index("idx_series_title", "title"),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
title = Column(String(512), nullable=False)
|
||||
title_normalized = Column(String(512), nullable=False)
|
||||
year = Column(SmallInteger, nullable=True)
|
||||
poster_url = Column(String(512), nullable=True)
|
||||
imdb_id = Column(String(20), nullable=True)
|
||||
added_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
seasons = relationship("Season", back_populates="series", cascade="all, delete-orphan")
|
||||
|
||||
|
||||
class Season(Base):
|
||||
__tablename__ = "seasons"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("series_id", "season_number", "nas_id", name="uq_season"),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
series_id = Column(Integer, ForeignKey("series.id", ondelete="CASCADE"), nullable=False)
|
||||
season_number = Column(SmallInteger, nullable=False)
|
||||
nas_id = Column(Integer, ForeignKey("nas_sources.id", ondelete="CASCADE"), nullable=False)
|
||||
|
||||
series = relationship("Series", back_populates="seasons")
|
||||
nas = relationship("NasSource", back_populates="seasons")
|
||||
episodes = relationship("Episode", back_populates="season", cascade="all, delete-orphan")
|
||||
|
||||
|
||||
class Episode(Base):
|
||||
__tablename__ = "episodes"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("file_path", name="uq_episode_path"),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
season_id = Column(Integer, ForeignKey("seasons.id", ondelete="CASCADE"), nullable=False)
|
||||
episode_number = Column(SmallInteger, nullable=False)
|
||||
title = Column(String(512), nullable=True)
|
||||
quality = Column(String(64), nullable=True)
|
||||
codec = Column(String(64), nullable=True)
|
||||
file_path = Column(String(2048), nullable=False)
|
||||
file_size = Column(BigInteger, nullable=True)
|
||||
added_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=False)
|
||||
|
||||
season = relationship("Season", back_populates="episodes")
|
||||
|
||||
|
||||
class UnmatchedFile(Base):
|
||||
__tablename__ = "unmatched_files"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
file_path = Column(String(2048), nullable=False)
|
||||
nas_id = Column(Integer, ForeignKey("nas_sources.id", ondelete="CASCADE"), nullable=False)
|
||||
reason = Column(String(512), nullable=False)
|
||||
scanned_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
|
||||
nas = relationship("NasSource", back_populates="unmatched")
|
||||
|
||||
|
||||
class ScanHistory(Base):
|
||||
__tablename__ = "scan_history"
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
scan_type = Column(Enum("full", "incremental"), nullable=False)
|
||||
status = Column(Enum("running", "completed", "failed"), nullable=False, default="running")
|
||||
started_at = Column(DateTime, default=datetime.utcnow, nullable=False)
|
||||
finished_at = Column(DateTime, nullable=True)
|
||||
files_found = Column(Integer, nullable=False, default=0)
|
||||
files_added = Column(Integer, nullable=False, default=0)
|
||||
files_removed = Column(Integer, nullable=False, default=0)
|
||||
files_unmatched = Column(Integer, nullable=False, default=0)
|
||||
error_message = Column(Text, nullable=True)
|
||||
|
||||
|
||||
class AltTitle(Base):
|
||||
__tablename__ = "alt_titles"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("media_type", "media_id", "lang", "title", name="uq_alt_title"),
|
||||
Index("idx_alt_title_search", "title"),
|
||||
)
|
||||
|
||||
id = Column(Integer, primary_key=True, autoincrement=True)
|
||||
media_type = Column(Enum("movie", "series"), nullable=False)
|
||||
media_id = Column(Integer, nullable=False)
|
||||
title = Column(String(512), nullable=False)
|
||||
lang = Column(String(10), nullable=False)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────
|
||||
# Engine / session factory
|
||||
# ─────────────────────────────────────────
|
||||
|
||||
_engine = None
|
||||
_SessionLocal = None
|
||||
|
||||
|
||||
def init_db(db_url: str) -> None:
|
||||
global _engine, _SessionLocal
|
||||
_engine = create_engine(
|
||||
db_url,
|
||||
pool_pre_ping=True,
|
||||
pool_recycle=3600,
|
||||
connect_args={"connect_timeout": 10},
|
||||
)
|
||||
_SessionLocal = sessionmaker(bind=_engine, autocommit=False, autoflush=False)
|
||||
logger.info("Database engine initialized")
|
||||
|
||||
|
||||
def get_session() -> Session:
|
||||
if _SessionLocal is None:
|
||||
raise RuntimeError("Database not initialized — call init_db() first")
|
||||
return _SessionLocal()
|
||||
|
||||
|
||||
def check_connection() -> bool:
|
||||
try:
|
||||
with _engine.connect() as conn:
|
||||
conn.execute(text("SELECT 1"))
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.error("Database connection check failed: %s", exc)
|
||||
return False
|
||||
125
backend/main.py
Normal file
125
backend/main.py
Normal file
@@ -0,0 +1,125 @@
|
||||
"""
|
||||
main.py — FastAPI application entry point.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import mimetypes
|
||||
import time
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI, HTTPException, Query
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from config import load_settings
|
||||
from database import check_connection, init_db, get_session
|
||||
from routers import scan, search, stats
|
||||
from scheduler import start_scheduler, stop_scheduler
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
||||
datefmt="%Y-%m-%dT%H:%M:%S",
|
||||
)
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_MAX_DB_RETRIES = 15
|
||||
_DB_RETRY_DELAY = 4
|
||||
|
||||
|
||||
def _migrate_db() -> None:
|
||||
"""Add columns introduced after initial schema creation."""
|
||||
session = get_session()
|
||||
migrations = [
|
||||
"ALTER TABLE movies ADD COLUMN poster_url VARCHAR(512) NULL",
|
||||
"ALTER TABLE series ADD COLUMN poster_url VARCHAR(512) NULL",
|
||||
"ALTER TABLE movies ADD COLUMN imdb_id VARCHAR(20) NULL",
|
||||
"ALTER TABLE series ADD COLUMN imdb_id VARCHAR(20) NULL",
|
||||
"""CREATE TABLE IF NOT EXISTS alt_titles (
|
||||
id INT UNSIGNED NOT NULL AUTO_INCREMENT,
|
||||
media_type ENUM('movie','series') NOT NULL,
|
||||
media_id INT UNSIGNED NOT NULL,
|
||||
title VARCHAR(512) NOT NULL,
|
||||
lang VARCHAR(10) NOT NULL,
|
||||
PRIMARY KEY (id),
|
||||
UNIQUE KEY uq_alt_title (media_type, media_id, lang, title(255)),
|
||||
INDEX idx_alt_title_search (title(255))
|
||||
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_unicode_ci""",
|
||||
]
|
||||
try:
|
||||
for sql in migrations:
|
||||
try:
|
||||
session.execute(__import__("sqlalchemy").text(sql))
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def _wait_for_db() -> None:
|
||||
for attempt in range(1, _MAX_DB_RETRIES + 1):
|
||||
if check_connection():
|
||||
logger.info("Database connection established")
|
||||
return
|
||||
logger.warning("Database not ready (attempt %d/%d) — retrying in %ds",
|
||||
attempt, _MAX_DB_RETRIES, _DB_RETRY_DELAY)
|
||||
time.sleep(_DB_RETRY_DELAY)
|
||||
raise RuntimeError("Could not connect to the database after multiple attempts")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
settings = load_settings()
|
||||
app.state.settings = settings
|
||||
|
||||
logger.info("Initializing database connection to %s:%s/%s",
|
||||
settings.db_host, settings.db_port, settings.db_name)
|
||||
init_db(settings.db_url)
|
||||
_wait_for_db()
|
||||
_migrate_db()
|
||||
|
||||
logger.info("Poster source: IMDB public suggestion API (no key required)")
|
||||
start_scheduler(settings)
|
||||
logger.info("MediaTracker backend ready")
|
||||
|
||||
yield
|
||||
|
||||
stop_scheduler()
|
||||
logger.info("MediaTracker backend shutdown")
|
||||
|
||||
|
||||
app = FastAPI(title="MediaTracker API", version="1.0.0", lifespan=lifespan)
|
||||
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=["*"],
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(search.router)
|
||||
app.include_router(scan.router)
|
||||
app.include_router(stats.router)
|
||||
|
||||
|
||||
@app.get("/api/health")
|
||||
def health():
|
||||
return {"status": "ok"}
|
||||
|
||||
|
||||
@app.get("/api/local-poster")
|
||||
def local_poster(path: str = Query(...)):
|
||||
"""
|
||||
Serve a local poster image file from the NAS.
|
||||
Path must be under /nas/ (read-only mount — no traversal possible outside).
|
||||
"""
|
||||
resolved = Path(path).resolve()
|
||||
nas_root = Path("/nas").resolve()
|
||||
if not str(resolved).startswith(str(nas_root)):
|
||||
raise HTTPException(status_code=403, detail="Access denied")
|
||||
if not resolved.is_file():
|
||||
raise HTTPException(status_code=404, detail="Poster file not found")
|
||||
mime, _ = mimetypes.guess_type(str(resolved))
|
||||
return FileResponse(str(resolved), media_type=mime or "image/jpeg")
|
||||
178
backend/metadata.py
Normal file
178
backend/metadata.py
Normal file
@@ -0,0 +1,178 @@
|
||||
"""
|
||||
metadata.py — Poster + metadata fetching.
|
||||
- Poster: IMDB public suggestion API (no key required)
|
||||
- Alt titles (FR/EN): Wikidata SPARQL (free, no registration)
|
||||
- Local poster detection (poster.jpg / folder.jpg alongside video)
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import urllib.error
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_REQUEST_TIMEOUT = 4
|
||||
_WIKIDATA_TIMEOUT = 8
|
||||
_LOCAL_POSTER_NAMES = ["poster.jpg", "poster.png", "folder.jpg", "folder.png",
|
||||
"cover.jpg", "cover.png", "movie.jpg"]
|
||||
_ALT_TITLE_LANGS = ("fr", "en", "de", "es")
|
||||
|
||||
|
||||
@dataclass
|
||||
class MediaMetadata:
|
||||
poster_url: Optional[str] = None
|
||||
imdb_id: Optional[str] = None
|
||||
|
||||
|
||||
# ─── IMDB suggestion API ─────────────────────────────────────────────────────
|
||||
|
||||
def _imdb_suggest(query: str) -> list[dict]:
|
||||
query_clean = query.lower().strip()
|
||||
if not query_clean:
|
||||
return []
|
||||
first_char = query_clean[0] if query_clean[0].isalpha() else "a"
|
||||
encoded = urllib.parse.quote(query_clean)
|
||||
url = f"https://sg.media-imdb.com/suggests/{first_char}/{encoded}.json"
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": "Mozilla/5.0",
|
||||
"Accept": "application/json, text/javascript",
|
||||
})
|
||||
with urllib.request.urlopen(req, timeout=_REQUEST_TIMEOUT) as resp:
|
||||
raw = resp.read().decode("utf-8")
|
||||
start = raw.index('(')
|
||||
end = raw.rindex(')')
|
||||
data = json.loads(raw[start + 1:end])
|
||||
return data.get("d", [])
|
||||
except (ValueError, KeyError):
|
||||
pass
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code != 404:
|
||||
logger.debug("IMDB suggest HTTP %d for %r", exc.code, query)
|
||||
except Exception as exc:
|
||||
logger.debug("IMDB suggest error for %r: %s", query, exc)
|
||||
return []
|
||||
|
||||
|
||||
def _extract_poster(item: dict) -> Optional[str]:
|
||||
img = item.get("i")
|
||||
if isinstance(img, list) and img:
|
||||
return str(img[0])
|
||||
if isinstance(img, dict):
|
||||
return img.get("imageUrl")
|
||||
return None
|
||||
|
||||
|
||||
def _best_match(items: list[dict], title: str, year: Optional[int], qid_filter: str) -> Optional[dict]:
|
||||
"""Return the best matching IMDB item dict, or None."""
|
||||
title_lower = title.lower()
|
||||
candidates = [it for it in items if it.get("qid") == qid_filter]
|
||||
if not candidates:
|
||||
return None
|
||||
if year:
|
||||
for it in candidates:
|
||||
if it.get("l", "").lower() == title_lower and it.get("y") == year:
|
||||
return it
|
||||
for it in candidates:
|
||||
if it.get("l", "").lower() == title_lower:
|
||||
return it
|
||||
# First candidate that has a poster
|
||||
for it in candidates:
|
||||
if _extract_poster(it):
|
||||
return it
|
||||
return candidates[0] if candidates else None
|
||||
|
||||
|
||||
def _get_metadata(title: str, year: Optional[int], qid_filter: str) -> MediaMetadata:
|
||||
items = _imdb_suggest(title)
|
||||
item = _best_match(items, title, year, qid_filter)
|
||||
if not item:
|
||||
return MediaMetadata()
|
||||
return MediaMetadata(
|
||||
poster_url=_extract_poster(item),
|
||||
imdb_id=item.get("id"),
|
||||
)
|
||||
|
||||
|
||||
def get_movie_metadata(title: str, year: Optional[int]) -> MediaMetadata:
|
||||
return _get_metadata(title, year, "movie")
|
||||
|
||||
|
||||
def get_series_metadata(title: str, year: Optional[int]) -> MediaMetadata:
|
||||
return _get_metadata(title, year, "tvSeries")
|
||||
|
||||
|
||||
# ─── Wikidata alt titles ─────────────────────────────────────────────────────
|
||||
|
||||
def get_alt_titles_batch(imdb_ids: list[str], retries: int = 3) -> dict[str, list[tuple[str, str]]]:
|
||||
"""
|
||||
Fetch FR/EN/DE/ES titles from Wikidata for a batch of IMDB IDs.
|
||||
Returns {imdb_id: [(lang, title), ...]}
|
||||
Retries up to `retries` times on 429/503 with exponential backoff.
|
||||
"""
|
||||
import time as _time
|
||||
|
||||
if not imdb_ids:
|
||||
return {}
|
||||
|
||||
values = " ".join(f'"{iid}"' for iid in imdb_ids)
|
||||
langs = " ".join(f'"{l}"' for l in _ALT_TITLE_LANGS)
|
||||
query = f"""
|
||||
SELECT ?imdbId ?label WHERE {{
|
||||
VALUES ?imdbId {{ {values} }}
|
||||
?item wdt:P345 ?imdbId .
|
||||
?item rdfs:label ?label .
|
||||
FILTER(LANG(?label) IN ({langs}))
|
||||
}}
|
||||
"""
|
||||
url = "https://query.wikidata.org/sparql?" + urllib.parse.urlencode({
|
||||
"query": query,
|
||||
"format": "json",
|
||||
})
|
||||
|
||||
for attempt in range(retries):
|
||||
try:
|
||||
req = urllib.request.Request(url, headers={
|
||||
"User-Agent": "MediaTracker/1.0",
|
||||
"Accept": "application/sparql-results+json",
|
||||
})
|
||||
with urllib.request.urlopen(req, timeout=_WIKIDATA_TIMEOUT) as resp:
|
||||
data = json.loads(resp.read().decode("utf-8"))
|
||||
|
||||
result: dict[str, list[tuple[str, str]]] = {}
|
||||
for row in data.get("results", {}).get("bindings", []):
|
||||
iid = row["imdbId"]["value"]
|
||||
label = row["label"]["value"]
|
||||
lang = row["label"].get("xml:lang", "")
|
||||
result.setdefault(iid, []).append((lang, label))
|
||||
return result
|
||||
|
||||
except urllib.error.HTTPError as exc:
|
||||
if exc.code in (429, 503) and attempt < retries - 1:
|
||||
wait = 5 * (2 ** attempt)
|
||||
logger.info("Wikidata rate-limited — retrying in %ds (attempt %d/%d)", wait, attempt + 1, retries)
|
||||
_time.sleep(wait)
|
||||
else:
|
||||
logger.warning("Wikidata batch failed (%d ids): HTTP %d", len(imdb_ids), exc.code)
|
||||
return {}
|
||||
except Exception as exc:
|
||||
logger.warning("Wikidata batch failed (%d ids): %s", len(imdb_ids), exc)
|
||||
return {}
|
||||
|
||||
return {}
|
||||
|
||||
|
||||
# ─── Local poster detection ───────────────────────────────────────────────────
|
||||
|
||||
def find_local_poster(video_path: Path) -> Optional[Path]:
|
||||
directory = video_path.parent
|
||||
for name in _LOCAL_POSTER_NAMES:
|
||||
candidate = directory / name
|
||||
if candidate.is_file():
|
||||
return candidate
|
||||
return None
|
||||
7
backend/requirements.txt
Normal file
7
backend/requirements.txt
Normal file
@@ -0,0 +1,7 @@
|
||||
fastapi==0.115.5
|
||||
uvicorn[standard]==0.32.1
|
||||
sqlalchemy==2.0.36
|
||||
mysqlclient==2.2.4
|
||||
parse-torrent-name==1.1.0
|
||||
apscheduler==3.10.4
|
||||
python-dotenv==1.0.1
|
||||
0
backend/routers/__init__.py
Normal file
0
backend/routers/__init__.py
Normal file
98
backend/routers/scan.py
Normal file
98
backend/routers/scan.py
Normal file
@@ -0,0 +1,98 @@
|
||||
from fastapi import APIRouter, HTTPException, Request
|
||||
|
||||
from database import ScanHistory, get_session
|
||||
from scheduler import (
|
||||
get_active_scan_id,
|
||||
is_scan_running,
|
||||
trigger_full_scan,
|
||||
trigger_incremental_scan,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/api/scan", tags=["scan"])
|
||||
|
||||
|
||||
@router.post("/full")
|
||||
def start_full_scan(request: Request):
|
||||
settings = request.app.state.settings
|
||||
if is_scan_running():
|
||||
raise HTTPException(status_code=409, detail="A scan is already running")
|
||||
scan_id = trigger_full_scan(settings)
|
||||
if scan_id is None:
|
||||
raise HTTPException(status_code=409, detail="A scan is already running")
|
||||
return {"scan_id": scan_id, "type": "full", "status": "running"}
|
||||
|
||||
|
||||
@router.post("/incremental")
|
||||
def start_incremental_scan(request: Request):
|
||||
settings = request.app.state.settings
|
||||
if is_scan_running():
|
||||
raise HTTPException(status_code=409, detail="A scan is already running")
|
||||
scan_id = trigger_incremental_scan(settings)
|
||||
if scan_id is None:
|
||||
raise HTTPException(status_code=409, detail="A scan is already running")
|
||||
return {"scan_id": scan_id, "type": "incremental", "status": "running"}
|
||||
|
||||
|
||||
@router.get("/status/{scan_id}")
|
||||
def get_scan_status(scan_id: int):
|
||||
session = get_session()
|
||||
try:
|
||||
record = session.query(ScanHistory).get(scan_id)
|
||||
if record is None:
|
||||
raise HTTPException(status_code=404, detail="Scan not found")
|
||||
return {
|
||||
"scan_id": record.id,
|
||||
"type": record.scan_type,
|
||||
"status": record.status,
|
||||
"started_at": record.started_at.isoformat() if record.started_at else None,
|
||||
"finished_at": record.finished_at.isoformat() if record.finished_at else None,
|
||||
"files_found": record.files_found,
|
||||
"files_added": record.files_added,
|
||||
"files_removed": record.files_removed,
|
||||
"files_unmatched": record.files_unmatched,
|
||||
"error_message": record.error_message,
|
||||
}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
@router.post("/fetch-alt-titles")
|
||||
def fetch_alt_titles_now():
|
||||
"""Trigger a Wikidata alt-title fetch in the background (no file scan)."""
|
||||
def _run():
|
||||
session = get_session()
|
||||
try:
|
||||
_fetch_alt_titles(session)
|
||||
finally:
|
||||
session.close()
|
||||
threading.Thread(target=_run, daemon=True, name="alt-titles-fetch").start()
|
||||
return {"status": "started"}
|
||||
|
||||
|
||||
@router.get("/history")
|
||||
def get_scan_history():
|
||||
session = get_session()
|
||||
try:
|
||||
records = (
|
||||
session.query(ScanHistory)
|
||||
.order_by(ScanHistory.started_at.desc())
|
||||
.limit(20)
|
||||
.all()
|
||||
)
|
||||
return [
|
||||
{
|
||||
"scan_id": r.id,
|
||||
"type": r.scan_type,
|
||||
"status": r.status,
|
||||
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||
"files_found": r.files_found,
|
||||
"files_added": r.files_added,
|
||||
"files_removed": r.files_removed,
|
||||
"files_unmatched": r.files_unmatched,
|
||||
"error_message": r.error_message,
|
||||
}
|
||||
for r in records
|
||||
]
|
||||
finally:
|
||||
session.close()
|
||||
102
backend/routers/search.py
Normal file
102
backend/routers/search.py
Normal file
@@ -0,0 +1,102 @@
|
||||
from fastapi import APIRouter, Query
|
||||
|
||||
from database import Movie, NasSource, Season, Series, get_session
|
||||
from metadata import _imdb_suggest
|
||||
|
||||
router = APIRouter(prefix="/api/search", tags=["search"])
|
||||
|
||||
_IMDB_QID = {"movie": "movie", "series": "tvSeries"}
|
||||
|
||||
|
||||
def _imdb_ids_for_query(q: str, kind: str) -> list[str]:
|
||||
"""Search IMDB with the user query and return matching IMDB IDs."""
|
||||
items = _imdb_suggest(q)
|
||||
qid = _IMDB_QID.get(kind)
|
||||
return [it["id"] for it in items if it.get("qid") == qid and it.get("id")]
|
||||
|
||||
|
||||
def _movie_matches(session, pattern: str, q: str) -> list:
|
||||
"""Return movies matching by title in DB, OR via IMDB cross-reference."""
|
||||
# Direct title match in DB
|
||||
by_title = session.query(Movie).filter(Movie.title.ilike(pattern)).all()
|
||||
found_ids = {m.id for m in by_title}
|
||||
|
||||
# Cross-language: search IMDB and look up matching IMDB IDs in our DB
|
||||
imdb_ids = _imdb_ids_for_query(q, "movie")
|
||||
if imdb_ids:
|
||||
cross = session.query(Movie).filter(
|
||||
Movie.imdb_id.in_(imdb_ids),
|
||||
Movie.id.notin_(found_ids) if found_ids else Movie.id.isnot(None),
|
||||
).all()
|
||||
by_title = by_title + cross
|
||||
|
||||
return sorted(by_title, key=lambda m: m.title)[:50]
|
||||
|
||||
|
||||
def _series_matches(session, pattern: str, q: str) -> list:
|
||||
"""Return series matching by title in DB, OR via IMDB cross-reference."""
|
||||
by_title = session.query(Series).filter(Series.title.ilike(pattern)).all()
|
||||
found_ids = {s.id for s in by_title}
|
||||
|
||||
imdb_ids = _imdb_ids_for_query(q, "series")
|
||||
if imdb_ids:
|
||||
cross = session.query(Series).filter(
|
||||
Series.imdb_id.in_(imdb_ids),
|
||||
Series.id.notin_(found_ids) if found_ids else Series.id.isnot(None),
|
||||
).all()
|
||||
by_title = by_title + cross
|
||||
|
||||
return sorted(by_title, key=lambda s: s.title)[:50]
|
||||
|
||||
|
||||
@router.get("")
|
||||
def search(
|
||||
q: str = Query(..., min_length=1),
|
||||
type: str = Query("all", pattern="^(all|movie|series)$"),
|
||||
):
|
||||
session = get_session()
|
||||
try:
|
||||
pattern = f"%{q}%"
|
||||
results = {"movies": [], "series": []}
|
||||
|
||||
if type in ("all", "movie"):
|
||||
for m in _movie_matches(session, pattern, q):
|
||||
nas = session.query(NasSource).get(m.nas_id)
|
||||
results["movies"].append({
|
||||
"id": m.id,
|
||||
"title": m.title,
|
||||
"year": m.year,
|
||||
"quality": m.quality,
|
||||
"codec": m.codec,
|
||||
"nas_name": nas.name if nas else None,
|
||||
"file_path": m.file_path,
|
||||
"file_size": m.file_size,
|
||||
"poster_url": m.poster_url,
|
||||
"imdb_id": m.imdb_id,
|
||||
})
|
||||
|
||||
if type in ("all", "series"):
|
||||
for s in _series_matches(session, pattern, q):
|
||||
seasons_data = []
|
||||
for season in sorted(s.seasons, key=lambda x: (x.season_number, x.nas_id)):
|
||||
nas = session.query(NasSource).get(season.nas_id)
|
||||
seasons_data.append({
|
||||
"season_number": season.season_number,
|
||||
"episodes_count": len(season.episodes),
|
||||
"nas_name": nas.name if nas else None,
|
||||
"nas_id": season.nas_id,
|
||||
})
|
||||
results["series"].append({
|
||||
"id": s.id,
|
||||
"title": s.title,
|
||||
"year": s.year,
|
||||
"seasons_count": len(s.seasons),
|
||||
"seasons": seasons_data,
|
||||
"poster_url": s.poster_url,
|
||||
"imdb_id": s.imdb_id,
|
||||
})
|
||||
|
||||
total = len(results["movies"]) + len(results["series"])
|
||||
return {"query": q, "results": results, "total": total}
|
||||
finally:
|
||||
session.close()
|
||||
56
backend/routers/stats.py
Normal file
56
backend/routers/stats.py
Normal file
@@ -0,0 +1,56 @@
|
||||
import os
|
||||
|
||||
from fastapi import APIRouter
|
||||
|
||||
from database import Episode, Movie, NasSource, ScanHistory, Season, Series, UnmatchedFile, get_session
|
||||
|
||||
router = APIRouter(prefix="/api", tags=["stats"])
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
def get_stats():
|
||||
session = get_session()
|
||||
try:
|
||||
movies_count = session.query(Movie).count()
|
||||
series_count = session.query(Series).count()
|
||||
episodes_count = session.query(Episode).count()
|
||||
nas_count = session.query(NasSource).count()
|
||||
unmatched_count = session.query(UnmatchedFile).count()
|
||||
|
||||
last_scan = (
|
||||
session.query(ScanHistory)
|
||||
.filter_by(status="completed")
|
||||
.order_by(ScanHistory.finished_at.desc())
|
||||
.first()
|
||||
)
|
||||
|
||||
return {
|
||||
"movies": movies_count,
|
||||
"series": series_count,
|
||||
"episodes": episodes_count,
|
||||
"nas_count": nas_count,
|
||||
"unmatched_files": unmatched_count,
|
||||
"last_scan": last_scan.finished_at.isoformat() if last_scan and last_scan.finished_at else None,
|
||||
}
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
@router.get("/nas")
|
||||
def get_nas_list():
|
||||
session = get_session()
|
||||
try:
|
||||
sources = session.query(NasSource).all()
|
||||
result = []
|
||||
for nas in sources:
|
||||
accessible = os.path.isdir(nas.path)
|
||||
result.append({
|
||||
"id": nas.id,
|
||||
"name": nas.name,
|
||||
"path": nas.path,
|
||||
"accessible": accessible,
|
||||
"last_scan": nas.last_scan.isoformat() if nas.last_scan else None,
|
||||
})
|
||||
return result
|
||||
finally:
|
||||
session.close()
|
||||
478
backend/scanner.py
Normal file
478
backend/scanner.py
Normal file
@@ -0,0 +1,478 @@
|
||||
"""
|
||||
scanner.py — Core media scanning logic.
|
||||
Identifies movies and TV series from NAS volumes using parse-torrent-name.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import unicodedata
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Optional
|
||||
|
||||
import PTN
|
||||
|
||||
from config import NasConfig, Settings
|
||||
from database import (
|
||||
Episode, Movie, NasSource, ScanHistory, Season, Series,
|
||||
UnmatchedFile, get_session,
|
||||
)
|
||||
from metadata import find_local_poster, get_movie_metadata, get_series_metadata
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Patterns that indicate a directory is a season folder
|
||||
_SEASON_DIR_RE = re.compile(r"(?i)(?:season|saison|^s)\s*(\d+)$")
|
||||
# Episode patterns in filename
|
||||
_EPISODE_FILE_RE = re.compile(r"(?i)S\d{2}E\d{2}|\d{1,2}x\d{2}")
|
||||
# Articles to strip during normalization
|
||||
_LEADING_ARTICLES = re.compile(r"^(?:the|le|la|les|a|an)\s+", re.IGNORECASE)
|
||||
# Characters to keep during normalization (letters, digits, space, hyphen)
|
||||
_KEEP_CHARS = re.compile(r"[^\w\s\-]", re.UNICODE)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────
|
||||
# Title normalization
|
||||
# ─────────────────────────────────────────
|
||||
|
||||
def normalize_title(title: str) -> str:
|
||||
title = title.lower().strip()
|
||||
title = unicodedata.normalize("NFKD", title)
|
||||
title = title.encode("ascii", "ignore").decode("ascii")
|
||||
title = _KEEP_CHARS.sub("", title)
|
||||
title = _LEADING_ARTICLES.sub("", title)
|
||||
title = re.sub(r"\s+", " ", title).strip()
|
||||
return title
|
||||
|
||||
|
||||
# ─────────────────────────────────────────
|
||||
# Series vs Movie detection
|
||||
# ─────────────────────────────────────────
|
||||
|
||||
def _is_season_directory(path: Path) -> Optional[int]:
|
||||
"""Return season number if any parent dir indicates a season folder, else None."""
|
||||
for part in path.parts:
|
||||
m = _SEASON_DIR_RE.search(part)
|
||||
if m:
|
||||
return int(m.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def classify_file(file_path: Path) -> tuple[bool, dict]:
|
||||
"""
|
||||
Parse the filename and classify as series (True) or movie (False).
|
||||
Returns (is_series, ptn_info).
|
||||
"""
|
||||
info = PTN.parse(file_path.stem)
|
||||
|
||||
# Priority 1: season directory name
|
||||
season_from_dir = _is_season_directory(file_path)
|
||||
if season_from_dir is not None:
|
||||
if "season" not in info:
|
||||
info["season"] = season_from_dir
|
||||
return True, info
|
||||
|
||||
# Priority 2: SxxExx pattern in filename
|
||||
if _EPISODE_FILE_RE.search(file_path.stem):
|
||||
return True, info
|
||||
|
||||
# Priority 3: PTN returned both season and episode
|
||||
if "season" in info and "episode" in info:
|
||||
return True, info
|
||||
|
||||
return False, info
|
||||
|
||||
|
||||
# ─────────────────────────────────────────
|
||||
# Database helpers
|
||||
# ─────────────────────────────────────────
|
||||
|
||||
def _upsert_nas_source(session, nas: NasConfig) -> NasSource:
|
||||
record = session.query(NasSource).filter_by(path=nas.container_path).first()
|
||||
if record is None:
|
||||
record = NasSource(name=nas.name, path=nas.container_path)
|
||||
session.add(record)
|
||||
session.flush()
|
||||
logger.info("Registered new NAS source: %s → %s", nas.name, nas.container_path)
|
||||
else:
|
||||
record.name = nas.name # update name if changed in .env
|
||||
return record
|
||||
|
||||
|
||||
def _get_or_create_series(session, title: str, year: Optional[int]) -> Series:
|
||||
normalized = normalize_title(title)
|
||||
record = session.query(Series).filter_by(title_normalized=normalized).first()
|
||||
if record is None:
|
||||
record = Series(title=title, title_normalized=normalized, year=year)
|
||||
session.add(record)
|
||||
session.flush()
|
||||
return record
|
||||
|
||||
|
||||
def _get_or_create_season(session, series: Series, season_number: int, nas_source: NasSource) -> Season:
|
||||
record = (
|
||||
session.query(Season)
|
||||
.filter_by(series_id=series.id, season_number=season_number, nas_id=nas_source.id)
|
||||
.first()
|
||||
)
|
||||
if record is None:
|
||||
record = Season(series_id=series.id, season_number=season_number, nas_id=nas_source.id)
|
||||
session.add(record)
|
||||
session.flush()
|
||||
return record
|
||||
|
||||
|
||||
# ─────────────────────────────────────────
|
||||
# File processing
|
||||
# ─────────────────────────────────────────
|
||||
|
||||
def _local_poster(file_path: Path) -> Optional[str]:
|
||||
local = find_local_poster(file_path)
|
||||
return f"local:{local}" if local else None
|
||||
|
||||
|
||||
def _fetch_posters_parallel(session) -> None:
|
||||
"""
|
||||
Fetch missing posters and/or IMDB IDs in parallel (10 workers).
|
||||
- Items without poster_url → fetch both poster + imdb_id
|
||||
- Items with poster_url but without imdb_id → fetch imdb_id only
|
||||
"""
|
||||
from sqlalchemy import or_
|
||||
all_items = (
|
||||
[(m.id, "movie", m.title, m.year, m.poster_url is None)
|
||||
for m in session.query(Movie).filter(
|
||||
or_(Movie.poster_url.is_(None), Movie.imdb_id.is_(None))
|
||||
).all()] +
|
||||
[(s.id, "series", s.title, s.year, s.poster_url is None)
|
||||
for s in session.query(Series).filter(
|
||||
or_(Series.poster_url.is_(None), Series.imdb_id.is_(None))
|
||||
).all()]
|
||||
)
|
||||
if not all_items:
|
||||
return
|
||||
|
||||
logger.info("Fetching metadata for %d items in parallel…", len(all_items))
|
||||
|
||||
def fetch_one(task):
|
||||
item_id, kind, title, year, need_poster = task
|
||||
if kind == "movie":
|
||||
meta = get_movie_metadata(title, year)
|
||||
else:
|
||||
meta = get_series_metadata(title, year)
|
||||
return item_id, kind, meta, need_poster
|
||||
|
||||
posters_fetched = 0
|
||||
with ThreadPoolExecutor(max_workers=10) as pool:
|
||||
futures = {pool.submit(fetch_one, t): t for t in all_items}
|
||||
for future in as_completed(futures):
|
||||
try:
|
||||
item_id, kind, meta, need_poster = future.result()
|
||||
updates = {}
|
||||
if need_poster and meta.poster_url:
|
||||
updates["poster_url"] = meta.poster_url
|
||||
posters_fetched += 1
|
||||
if meta.imdb_id:
|
||||
updates["imdb_id"] = meta.imdb_id
|
||||
if updates:
|
||||
model = Movie if kind == "movie" else Series
|
||||
session.query(model).filter_by(id=item_id).update(updates)
|
||||
except Exception as exc:
|
||||
logger.debug("Metadata fetch error: %s", exc)
|
||||
|
||||
session.commit()
|
||||
logger.info("Metadata pass done — new posters: %d, total processed: %d",
|
||||
posters_fetched, len(all_items))
|
||||
|
||||
|
||||
|
||||
def _process_file(
|
||||
session,
|
||||
file_path: Path,
|
||||
nas_source: NasSource,
|
||||
settings: Settings,
|
||||
counters: dict,
|
||||
poster_cache: dict,
|
||||
) -> None:
|
||||
str_path = str(file_path)
|
||||
|
||||
try:
|
||||
file_size = file_path.stat().st_size
|
||||
except OSError:
|
||||
file_size = None
|
||||
|
||||
try:
|
||||
is_series, info = classify_file(file_path)
|
||||
except Exception as exc:
|
||||
reason = f"PTN parse error: {exc}"
|
||||
logger.warning("Unmatched %s — %s", str_path, reason)
|
||||
_record_unmatched(session, str_path, nas_source.id, reason)
|
||||
counters["unmatched"] += 1
|
||||
return
|
||||
|
||||
title = info.get("title")
|
||||
if not title:
|
||||
reason = "No title extracted by PTN"
|
||||
_record_unmatched(session, str_path, nas_source.id, reason)
|
||||
counters["unmatched"] += 1
|
||||
return
|
||||
|
||||
year = info.get("year")
|
||||
quality = info.get("quality")
|
||||
codec = info.get("codec")
|
||||
|
||||
# Fallback: extract year from title like "Black Bag (2025)" when PTN misses it
|
||||
if not year:
|
||||
m = re.search(r'\((\d{4})\)\s*$', title)
|
||||
if m:
|
||||
year = int(m.group(1))
|
||||
title = title[:m.start()].strip()
|
||||
|
||||
if is_series:
|
||||
season_num = info.get("season", 1)
|
||||
episode_num = info.get("episode")
|
||||
if episode_num is None:
|
||||
reason = "Series detected but no episode number found"
|
||||
_record_unmatched(session, str_path, nas_source.id, reason)
|
||||
counters["unmatched"] += 1
|
||||
return
|
||||
|
||||
series = _get_or_create_series(session, title, year)
|
||||
|
||||
if series.poster_url is None:
|
||||
local = _local_poster(file_path)
|
||||
if local:
|
||||
series.poster_url = local
|
||||
|
||||
season = _get_or_create_season(session, series, int(season_num), nas_source)
|
||||
|
||||
existing = session.query(Episode).filter_by(file_path=str_path).first()
|
||||
if existing is None:
|
||||
episode = Episode(
|
||||
season_id=season.id,
|
||||
episode_number=int(episode_num),
|
||||
title=info.get("episode_title"),
|
||||
quality=quality,
|
||||
codec=codec,
|
||||
file_path=str_path,
|
||||
file_size=file_size,
|
||||
)
|
||||
session.add(episode)
|
||||
counters["added"] += 1
|
||||
else:
|
||||
existing.quality = quality
|
||||
existing.codec = codec
|
||||
existing.file_size = file_size
|
||||
existing.updated_at = datetime.utcnow()
|
||||
else:
|
||||
existing = session.query(Movie).filter_by(file_path=str_path).first()
|
||||
if existing is None:
|
||||
movie = Movie(
|
||||
title=title,
|
||||
year=year,
|
||||
quality=quality,
|
||||
codec=codec,
|
||||
file_path=str_path,
|
||||
file_size=file_size,
|
||||
poster_url=_local_poster(file_path),
|
||||
nas_id=nas_source.id,
|
||||
)
|
||||
session.add(movie)
|
||||
counters["added"] += 1
|
||||
else:
|
||||
existing.title = title
|
||||
existing.year = year
|
||||
existing.quality = quality
|
||||
existing.codec = codec
|
||||
existing.file_size = file_size
|
||||
existing.updated_at = datetime.utcnow()
|
||||
if existing.poster_url is None:
|
||||
local = _local_poster(file_path)
|
||||
if local:
|
||||
existing.poster_url = local
|
||||
|
||||
counters["found"] += 1
|
||||
|
||||
|
||||
def _record_unmatched(session, file_path: str, nas_id: int, reason: str) -> None:
|
||||
record = UnmatchedFile(file_path=file_path, nas_id=nas_id, reason=reason)
|
||||
session.add(record)
|
||||
|
||||
|
||||
# ─────────────────────────────────────────
|
||||
# Staleness checks (removal of missing files)
|
||||
# ─────────────────────────────────────────
|
||||
|
||||
def _remove_stale_movies(session, nas_source: NasSource) -> int:
|
||||
removed = 0
|
||||
movies = session.query(Movie).filter_by(nas_id=nas_source.id).all()
|
||||
for movie in movies:
|
||||
if not Path(movie.file_path).exists():
|
||||
logger.info("Removing stale movie: %s", movie.file_path)
|
||||
session.delete(movie)
|
||||
removed += 1
|
||||
return removed
|
||||
|
||||
|
||||
def _remove_stale_episodes(session, nas_source: NasSource) -> int:
|
||||
removed = 0
|
||||
seasons = session.query(Season).filter_by(nas_id=nas_source.id).all()
|
||||
for season in seasons:
|
||||
for episode in list(season.episodes):
|
||||
if not Path(episode.file_path).exists():
|
||||
logger.info("Removing stale episode: %s", episode.file_path)
|
||||
session.delete(episode)
|
||||
removed += 1
|
||||
# Prune empty seasons
|
||||
session.flush()
|
||||
if not season.episodes:
|
||||
# Re-query to check after flush
|
||||
refreshed = session.query(Season).get(season.id)
|
||||
if refreshed and not refreshed.episodes:
|
||||
session.delete(refreshed)
|
||||
# Prune series with no remaining seasons
|
||||
for series in session.query(Series).all():
|
||||
session.flush()
|
||||
refreshed = session.query(Series).get(series.id)
|
||||
if refreshed and not refreshed.seasons:
|
||||
session.delete(refreshed)
|
||||
return removed
|
||||
|
||||
|
||||
# ─────────────────────────────────────────
|
||||
# Public scan functions
|
||||
# ─────────────────────────────────────────
|
||||
|
||||
def run_full_scan(settings: Settings, scan_id: int) -> None:
|
||||
logger.info("Starting full scan (scan_id=%d)", scan_id)
|
||||
session = get_session()
|
||||
counters = {"found": 0, "added": 0, "removed": 0, "unmatched": 0}
|
||||
poster_cache: dict = {}
|
||||
|
||||
try:
|
||||
# Clear unmatched from previous scans
|
||||
session.query(UnmatchedFile).delete()
|
||||
|
||||
for nas in settings.nas_list:
|
||||
if not os.path.isdir(nas.container_path):
|
||||
logger.warning("NAS path not accessible: %s — skipping", nas.container_path)
|
||||
continue
|
||||
|
||||
nas_source = _upsert_nas_source(session, nas)
|
||||
session.commit()
|
||||
|
||||
logger.info("Scanning NAS: %s (%s)", nas.name, nas.container_path)
|
||||
|
||||
for root, _dirs, files in os.walk(nas.container_path):
|
||||
for filename in files:
|
||||
ext = Path(filename).suffix.lower()
|
||||
if ext not in settings.video_extensions:
|
||||
continue
|
||||
file_path = Path(root) / filename
|
||||
_process_file(session, file_path, nas_source, settings, counters, poster_cache)
|
||||
|
||||
removed = _remove_stale_movies(session, nas_source)
|
||||
removed += _remove_stale_episodes(session, nas_source)
|
||||
counters["removed"] += removed
|
||||
|
||||
nas_source.last_scan = datetime.utcnow()
|
||||
session.commit()
|
||||
|
||||
_fetch_posters_parallel(session)
|
||||
_finish_scan(session, scan_id, "completed", counters)
|
||||
logger.info("Full scan complete: %s", counters)
|
||||
|
||||
except Exception as exc:
|
||||
logger.exception("Full scan failed: %s", exc)
|
||||
_finish_scan(session, scan_id, "failed", counters, str(exc))
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def run_incremental_scan(settings: Settings, scan_id: int) -> None:
|
||||
logger.info("Starting incremental scan (scan_id=%d)", scan_id)
|
||||
session = get_session()
|
||||
counters = {"found": 0, "added": 0, "removed": 0, "unmatched": 0}
|
||||
poster_cache: dict = {}
|
||||
|
||||
try:
|
||||
for nas in settings.nas_list:
|
||||
if not os.path.isdir(nas.container_path):
|
||||
logger.warning("NAS path not accessible: %s — skipping", nas.container_path)
|
||||
continue
|
||||
|
||||
nas_source = _upsert_nas_source(session, nas)
|
||||
session.commit()
|
||||
|
||||
since = nas_source.last_scan
|
||||
logger.info(
|
||||
"Incremental scan NAS: %s — changes since %s",
|
||||
nas.name,
|
||||
since.isoformat() if since else "never",
|
||||
)
|
||||
|
||||
since_ts = since.timestamp() if since else 0.0
|
||||
|
||||
for root, _dirs, files in os.walk(nas.container_path):
|
||||
for filename in files:
|
||||
ext = Path(filename).suffix.lower()
|
||||
if ext not in settings.video_extensions:
|
||||
continue
|
||||
file_path = Path(root) / filename
|
||||
try:
|
||||
mtime = file_path.stat().st_mtime
|
||||
except OSError:
|
||||
continue
|
||||
if mtime > since_ts:
|
||||
_process_file(session, file_path, nas_source, settings, counters, poster_cache)
|
||||
|
||||
removed = _remove_stale_movies(session, nas_source)
|
||||
removed += _remove_stale_episodes(session, nas_source)
|
||||
counters["removed"] += removed
|
||||
|
||||
nas_source.last_scan = datetime.utcnow()
|
||||
session.commit()
|
||||
|
||||
_fetch_posters_parallel(session)
|
||||
_finish_scan(session, scan_id, "completed", counters)
|
||||
logger.info("Incremental scan complete: %s", counters)
|
||||
|
||||
except Exception as exc:
|
||||
logger.exception("Incremental scan failed: %s", exc)
|
||||
_finish_scan(session, scan_id, "failed", counters, str(exc))
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def _finish_scan(session, scan_id: int, status: str, counters: dict, error: str = None) -> None:
|
||||
record = session.query(ScanHistory).get(scan_id)
|
||||
if record:
|
||||
record.status = status
|
||||
record.finished_at = datetime.utcnow()
|
||||
record.files_found = counters["found"]
|
||||
record.files_added = counters["added"]
|
||||
record.files_removed = counters["removed"]
|
||||
record.files_unmatched = counters["unmatched"]
|
||||
record.error_message = error
|
||||
session.commit()
|
||||
|
||||
|
||||
def create_scan_record(scan_type: str) -> int:
|
||||
"""Insert a new running scan record and return its ID."""
|
||||
session = get_session()
|
||||
try:
|
||||
record = ScanHistory(scan_type=scan_type, status="running", started_at=datetime.utcnow())
|
||||
session.add(record)
|
||||
session.commit()
|
||||
return record.id
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
|
||||
def has_any_scan_completed() -> bool:
|
||||
session = get_session()
|
||||
try:
|
||||
return session.query(ScanHistory).filter_by(status="completed").count() > 0
|
||||
finally:
|
||||
session.close()
|
||||
95
backend/scheduler.py
Normal file
95
backend/scheduler.py
Normal file
@@ -0,0 +1,95 @@
|
||||
"""
|
||||
scheduler.py — Background scan scheduler using APScheduler.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import threading
|
||||
|
||||
from apscheduler.schedulers.background import BackgroundScheduler
|
||||
|
||||
from config import Settings
|
||||
from scanner import (
|
||||
create_scan_record,
|
||||
has_any_scan_completed,
|
||||
run_full_scan,
|
||||
run_incremental_scan,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_scheduler: BackgroundScheduler | None = None
|
||||
_scan_lock = threading.Lock()
|
||||
_active_scan_id: int | None = None
|
||||
|
||||
|
||||
def get_active_scan_id() -> int | None:
|
||||
return _active_scan_id
|
||||
|
||||
|
||||
def is_scan_running() -> bool:
|
||||
return _active_scan_id is not None
|
||||
|
||||
|
||||
def _run_scan_protected(scan_type: str, settings: Settings) -> int | None:
|
||||
"""Try to acquire lock and run a scan. Returns scan_id or None if already running."""
|
||||
global _active_scan_id
|
||||
if not _scan_lock.acquire(blocking=False):
|
||||
logger.warning("Scan already in progress — skipping %s scan request", scan_type)
|
||||
return None
|
||||
|
||||
scan_id = create_scan_record(scan_type)
|
||||
_active_scan_id = scan_id
|
||||
logger.info("Scan %s started (id=%d)", scan_type, scan_id)
|
||||
|
||||
def _run():
|
||||
global _active_scan_id
|
||||
try:
|
||||
if scan_type == "full":
|
||||
run_full_scan(settings, scan_id)
|
||||
else:
|
||||
run_incremental_scan(settings, scan_id)
|
||||
finally:
|
||||
_active_scan_id = None
|
||||
_scan_lock.release()
|
||||
logger.info("Scan %s finished (id=%d)", scan_type, scan_id)
|
||||
|
||||
thread = threading.Thread(target=_run, daemon=True, name=f"scan-{scan_type}-{scan_id}")
|
||||
thread.start()
|
||||
return scan_id
|
||||
|
||||
|
||||
def trigger_full_scan(settings: Settings) -> int | None:
|
||||
return _run_scan_protected("full", settings)
|
||||
|
||||
|
||||
def trigger_incremental_scan(settings: Settings) -> int | None:
|
||||
return _run_scan_protected("incremental", settings)
|
||||
|
||||
|
||||
def start_scheduler(settings: Settings) -> None:
|
||||
global _scheduler
|
||||
|
||||
_scheduler = BackgroundScheduler(timezone="UTC")
|
||||
_scheduler.add_job(
|
||||
func=lambda: trigger_incremental_scan(settings),
|
||||
trigger="interval",
|
||||
hours=settings.scan_interval_hours,
|
||||
id="incremental_scan",
|
||||
replace_existing=True,
|
||||
)
|
||||
_scheduler.start()
|
||||
logger.info(
|
||||
"Scheduler started — incremental scan every %d hour(s)",
|
||||
settings.scan_interval_hours,
|
||||
)
|
||||
|
||||
# Trigger initial full scan if the database has never been scanned
|
||||
if not has_any_scan_completed():
|
||||
logger.info("No prior completed scan found — launching initial full scan")
|
||||
trigger_full_scan(settings)
|
||||
|
||||
|
||||
def stop_scheduler() -> None:
|
||||
if _scheduler and _scheduler.running:
|
||||
_scheduler.shutdown(wait=False)
|
||||
logger.info("Scheduler stopped")
|
||||
Reference in New Issue
Block a user