init
This commit is contained in:
@@ -0,0 +1,7 @@
|
|||||||
|
.env
|
||||||
|
.git
|
||||||
|
.gitignore
|
||||||
|
data
|
||||||
|
catalog-cache
|
||||||
|
README.md
|
||||||
|
scripts
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
# Exact browser origin allowed to request HLS resources. This value is required.
|
||||||
|
PLAYER_ORIGIN=http://192.168.1.20:3000
|
||||||
|
|
||||||
|
# Development uses the repository test library. Production must use an absolute host path.
|
||||||
|
MEDIA_DIR=./data
|
||||||
|
|
||||||
|
# Persistent host directory for catalog JSON snapshots and generated JPEG covers.
|
||||||
|
# Create it before starting the stack.
|
||||||
|
CATALOG_CACHE_DIR=./catalog-cache
|
||||||
|
CATALOG_REFRESH_INTERVAL_SECONDS=21600
|
||||||
|
|
||||||
|
# LAN HTTP listener published by Docker Compose.
|
||||||
|
HLS_BIND_ADDRESS=0.0.0.0
|
||||||
|
HLS_PORT=8124
|
||||||
|
|
||||||
|
# Fixed build inputs. Upgrade these deliberately and re-run the smoke test.
|
||||||
|
NGINX_VERSION=1.30.5
|
||||||
|
KALTURA_VOD_REF=26f06877b0f2a2336e59cda93a3de18d7b23a3e2
|
||||||
|
IMAGE_NAME=nginx-hls-vod
|
||||||
+30
@@ -0,0 +1,30 @@
|
|||||||
|
# Local configuration and runtime catalog cache
|
||||||
|
.env
|
||||||
|
.env.*
|
||||||
|
!.env.example
|
||||||
|
catalog-cache/
|
||||||
|
|
||||||
|
# Python caches, virtual environments, and test coverage
|
||||||
|
__pycache__/
|
||||||
|
*.py[cod]
|
||||||
|
.pytest_cache/
|
||||||
|
.coverage
|
||||||
|
coverage.xml
|
||||||
|
htmlcov/
|
||||||
|
.venv/
|
||||||
|
venv/
|
||||||
|
env/
|
||||||
|
|
||||||
|
# Local Docker overrides and editor/OS files
|
||||||
|
compose.override.yml
|
||||||
|
compose.override.yaml
|
||||||
|
docker-compose.override.yml
|
||||||
|
*.log
|
||||||
|
.DS_Store
|
||||||
|
Thumbs.db
|
||||||
|
.idea/
|
||||||
|
*.iml
|
||||||
|
|
||||||
|
# Test media stays local and is never sent as a Docker build context.
|
||||||
|
data/*
|
||||||
|
!data/.gitkeep
|
||||||
+96
@@ -0,0 +1,96 @@
|
|||||||
|
FROM debian:bookworm-slim AS builder
|
||||||
|
|
||||||
|
ARG NGINX_VERSION=1.30.5
|
||||||
|
ARG KALTURA_VOD_REF=26f06877b0f2a2336e59cda93a3de18d7b23a3e2
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
ca-certificates \
|
||||||
|
curl \
|
||||||
|
libpcre2-dev \
|
||||||
|
zlib1g-dev \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
RUN mkdir -p /src/nginx-vod-module \
|
||||||
|
&& curl -fsSL "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" -o /tmp/nginx.tar.gz \
|
||||||
|
&& curl -fsSL "https://codeload.github.com/kaltura/nginx-vod-module/tar.gz/${KALTURA_VOD_REF}" -o /tmp/nginx-vod-module.tar.gz \
|
||||||
|
&& tar -xzf /tmp/nginx.tar.gz -C /src \
|
||||||
|
&& tar -xzf /tmp/nginx-vod-module.tar.gz -C /src/nginx-vod-module --strip-components=1 \
|
||||||
|
&& rm -f /tmp/nginx.tar.gz /tmp/nginx-vod-module.tar.gz
|
||||||
|
|
||||||
|
WORKDIR /src/nginx-${NGINX_VERSION}
|
||||||
|
|
||||||
|
RUN ./configure \
|
||||||
|
--prefix=/usr/share/nginx \
|
||||||
|
--sbin-path=/usr/sbin/nginx \
|
||||||
|
--conf-path=/etc/nginx/nginx.conf \
|
||||||
|
--pid-path=/tmp/nginx.pid \
|
||||||
|
--error-log-path=/dev/stderr \
|
||||||
|
--http-log-path=/dev/stdout \
|
||||||
|
--http-client-body-temp-path=/tmp/client_body \
|
||||||
|
--http-proxy-temp-path=/tmp/proxy \
|
||||||
|
--http-fastcgi-temp-path=/tmp/fastcgi \
|
||||||
|
--http-uwsgi-temp-path=/tmp/uwsgi \
|
||||||
|
--http-scgi-temp-path=/tmp/scgi \
|
||||||
|
--with-file-aio \
|
||||||
|
--with-threads \
|
||||||
|
--with-http_gzip_static_module \
|
||||||
|
--with-http_stub_status_module \
|
||||||
|
--with-cc-opt='-O2' \
|
||||||
|
--add-module=/src/nginx-vod-module \
|
||||||
|
&& make -j"$(nproc)" \
|
||||||
|
&& make install
|
||||||
|
|
||||||
|
FROM debian:bookworm-slim AS vod
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
ca-certificates \
|
||||||
|
curl \
|
||||||
|
gettext-base \
|
||||||
|
libpcre2-8-0 \
|
||||||
|
zlib1g \
|
||||||
|
&& addgroup --system nginx \
|
||||||
|
&& adduser --system --ingroup nginx --no-create-home --home /nonexistent nginx \
|
||||||
|
&& mkdir -p /etc/nginx/templates \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY --from=builder /usr/sbin/nginx /usr/sbin/nginx
|
||||||
|
COPY --from=builder /etc/nginx/mime.types /etc/nginx/mime.types
|
||||||
|
COPY config/nginx.conf.template /etc/nginx/templates/nginx.conf.template
|
||||||
|
COPY docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||||
|
|
||||||
|
RUN chmod 0755 /usr/local/bin/docker-entrypoint.sh
|
||||||
|
|
||||||
|
USER nginx
|
||||||
|
|
||||||
|
EXPOSE 8080
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
|
||||||
|
CMD curl --fail --silent http://127.0.0.1:8080/healthz || exit 1
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
||||||
|
|
||||||
|
FROM debian:bookworm-slim AS catalog
|
||||||
|
|
||||||
|
RUN apt-get update \
|
||||||
|
&& apt-get install -y --no-install-recommends \
|
||||||
|
ffmpeg \
|
||||||
|
python3 \
|
||||||
|
&& addgroup --system catalog \
|
||||||
|
&& adduser --system --ingroup catalog --no-create-home --home /nonexistent catalog \
|
||||||
|
&& mkdir -p /app /cache \
|
||||||
|
&& chown catalog:catalog /app /cache \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
COPY catalog/app.py /app/app.py
|
||||||
|
|
||||||
|
USER catalog
|
||||||
|
|
||||||
|
EXPOSE 8081
|
||||||
|
|
||||||
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||||
|
CMD python3 -c "from urllib.request import urlopen; urlopen('http://127.0.0.1:8081/healthz', timeout=3).read()" || exit 1
|
||||||
|
|
||||||
|
ENTRYPOINT ["python3", "/app/app.py"]
|
||||||
@@ -0,0 +1,159 @@
|
|||||||
|
# NGINX HLS VOD
|
||||||
|
|
||||||
|
Docker Compose deployment for local MP4 to HLS VOD packaging using Kaltura
|
||||||
|
`nginx-vod-module`. The service maps a media tree read-only into the container
|
||||||
|
and preserves its hierarchy in HLS URLs.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
- ARM64 Linux Docker host
|
||||||
|
- Trusted LAN HTTP service on port `8124`
|
||||||
|
- One H.264 video track and one AAC audio track per MP4
|
||||||
|
- One configured browser origin for CORS
|
||||||
|
- No authentication, TLS, ABR, external subtitles, or multiple audio/video
|
||||||
|
tracks
|
||||||
|
|
||||||
|
An attached-picture video stream such as MP4 cover art is ignored by the media
|
||||||
|
validator.
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
Copy the environment template and replace the example player origin:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cp .env.example .env
|
||||||
|
```
|
||||||
|
|
||||||
|
Required values:
|
||||||
|
|
||||||
|
```dotenv
|
||||||
|
PLAYER_ORIGIN=http://192.168.1.20:3000
|
||||||
|
MEDIA_DIR=/srv/media
|
||||||
|
CATALOG_CACHE_DIR=/srv/nginx-hls/catalog-cache
|
||||||
|
HLS_PORT=8124
|
||||||
|
```
|
||||||
|
|
||||||
|
`MEDIA_DIR=./data` is appropriate only for local development. Production uses
|
||||||
|
an absolute path on the Docker host. The container user must be able to read
|
||||||
|
the mounted files and directories. `CATALOG_CACHE_DIR` stores catalog snapshots
|
||||||
|
and generated covers. Create it before startup, for example
|
||||||
|
`install -d -m 0700 /srv/nginx-hls/catalog-cache`.
|
||||||
|
`CATALOG_REFRESH_INTERVAL_SECONDS` defaults to `21600` (6 hours).
|
||||||
|
|
||||||
|
The Dockerfile deliberately fixes NGINX `1.30.5` and Kaltura module commit
|
||||||
|
`26f06877b0f2a2336e59cda93a3de18d7b23a3e2`. Change either only as a deliberate
|
||||||
|
upgrade, then rebuild and run the smoke test.
|
||||||
|
|
||||||
|
## Run
|
||||||
|
|
||||||
|
Start the stack after creating the configured cache directory:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
set -a
|
||||||
|
. ./.env
|
||||||
|
set +a
|
||||||
|
install -d -m 0700 "$CATALOG_CACHE_DIR"
|
||||||
|
docker compose --env-file .env up -d --build
|
||||||
|
docker compose ps
|
||||||
|
```
|
||||||
|
|
||||||
|
The catalog validates files during its background refresh and omits invalid
|
||||||
|
MP4s from the published snapshot. Use `./scripts/validate-media.sh "$MEDIA_DIR"`
|
||||||
|
before publishing only when an update must contain no invalid files at all.
|
||||||
|
|
||||||
|
For a media file at:
|
||||||
|
|
||||||
|
```text
|
||||||
|
/srv/media/movies/example.mp4
|
||||||
|
```
|
||||||
|
|
||||||
|
request this HLS master playlist:
|
||||||
|
|
||||||
|
```text
|
||||||
|
http://<vod-host>:8124/hls/movies/example.mp4/master.m3u8
|
||||||
|
```
|
||||||
|
|
||||||
|
Path segments must be URL encoded by clients. There is intentionally no
|
||||||
|
directory listing or progressive-download route.
|
||||||
|
|
||||||
|
## Media Catalog
|
||||||
|
|
||||||
|
Clients discover validated MP4 files through the same origin:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/media
|
||||||
|
```
|
||||||
|
|
||||||
|
The response is a flat array from the latest completed catalog snapshot:
|
||||||
|
|
||||||
|
```json
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"path": "movies/example.mp4",
|
||||||
|
"name": "example",
|
||||||
|
"playlistUrl": "/hls/movies/example.mp4/master.m3u8",
|
||||||
|
"coverUrl": "/api/media/covers/<cache-key>.jpg"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
```
|
||||||
|
|
||||||
|
Catalog requests never scan media or run FFmpeg. A background worker scans the
|
||||||
|
tree at startup and then every six hours, validates one H.264 video track plus
|
||||||
|
one AAC audio track, and pre-generates covers for every valid file. It publishes
|
||||||
|
the completed JSON and JPEGs atomically, so clients continue to read the last
|
||||||
|
successful snapshot while a refresh runs or fails. Without any completed
|
||||||
|
snapshot, media and cover requests return `503`.
|
||||||
|
|
||||||
|
Use the `coverUrl` from the media item to request a pre-generated cover:
|
||||||
|
|
||||||
|
```text
|
||||||
|
GET /api/media/covers/<cache-key>.jpg
|
||||||
|
```
|
||||||
|
|
||||||
|
`GET /api/media/status` reports the refresh state, latest successful timestamp,
|
||||||
|
valid and skipped file counts, and the latest build error. Trigger an immediate
|
||||||
|
asynchronous rebuild with `POST /api/media/rescan`; concurrent requests do not
|
||||||
|
start duplicate builds. The service is intended for a trusted LAN, so this
|
||||||
|
endpoint has no additional authentication. Browser caching remains disabled.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
curl -sS http://<vod-host>:8124/api/media/status
|
||||||
|
curl -i -X POST http://<vod-host>:8124/api/media/rescan
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the smoke test after the container becomes healthy. Its default test media
|
||||||
|
is `data/aa/1170193193-1-192.mp4`; override it with a relative, URL-encoded
|
||||||
|
`TEST_MEDIA_PATH` when needed.
|
||||||
|
|
||||||
|
```sh
|
||||||
|
export PLAYER_ORIGIN=http://192.168.1.20:3000
|
||||||
|
./scripts/smoke-test.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
## Publishing Updates
|
||||||
|
|
||||||
|
Use a staging directory on the same filesystem as the media library, validate
|
||||||
|
the staged MP4, and atomically rename it into place. Restart the container only
|
||||||
|
after the rename completes:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
./scripts/validate-media.sh /srv/media/.staging
|
||||||
|
mv -T /srv/media/.staging/example.mp4 /srv/media/movies/example.mp4
|
||||||
|
./scripts/refresh-vod.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
The service sends `Cache-Control: no-store` and the restart clears VOD caches.
|
||||||
|
This makes a replacement visible promptly, but intentionally interrupts active
|
||||||
|
playback for a short period. Do not overwrite MP4 files in place.
|
||||||
|
|
||||||
|
## Operations
|
||||||
|
|
||||||
|
- Health endpoint: `GET /healthz`
|
||||||
|
- Logs: `docker compose logs -f vod`
|
||||||
|
- Stop: `docker compose down`
|
||||||
|
- The host firewall should limit TCP `8124` to the trusted LAN. This service
|
||||||
|
does not provide TLS or authentication.
|
||||||
|
|
||||||
|
The first build downloads the fixed source inputs and base image. The ARM64
|
||||||
|
target host therefore needs access to Docker registries, `nginx.org`, and
|
||||||
|
GitHub during `docker compose build`.
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
"""Media catalog service package."""
|
||||||
+475
@@ -0,0 +1,475 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""Prebuild and serve a read-only MP4 catalog snapshot."""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import shutil
|
||||||
|
import subprocess
|
||||||
|
import threading
|
||||||
|
import uuid
|
||||||
|
from dataclasses import dataclass
|
||||||
|
from datetime import UTC, datetime
|
||||||
|
from http import HTTPStatus
|
||||||
|
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||||
|
from pathlib import Path
|
||||||
|
from urllib.parse import quote, urlsplit
|
||||||
|
|
||||||
|
|
||||||
|
LOG = logging.getLogger(__name__)
|
||||||
|
COVER_NAME = re.compile(r"^[0-9a-f]{64}\.jpg$")
|
||||||
|
DEFAULT_REFRESH_INTERVAL_SECONDS = 6 * 60 * 60
|
||||||
|
|
||||||
|
|
||||||
|
@dataclass(frozen=True)
|
||||||
|
class ProbeResult:
|
||||||
|
duration: float
|
||||||
|
attached_picture_stream: int | None
|
||||||
|
|
||||||
|
|
||||||
|
def inspect_media(source: Path) -> ProbeResult | None:
|
||||||
|
"""Validate a source file against the HLS service's supported streams."""
|
||||||
|
command = [
|
||||||
|
"ffprobe",
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-show_entries",
|
||||||
|
"format=duration:stream=index,codec_name,codec_type:stream_disposition=attached_pic",
|
||||||
|
"-of",
|
||||||
|
"json",
|
||||||
|
str(source),
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
result = subprocess.run(command, capture_output=True, check=False, text=True, timeout=30)
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as error:
|
||||||
|
LOG.warning("Unable to inspect %s: %s", source, error)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if result.returncode != 0:
|
||||||
|
LOG.warning("Ignoring unreadable media %s: %s", source, result.stderr.strip())
|
||||||
|
return None
|
||||||
|
|
||||||
|
try:
|
||||||
|
payload = json.loads(result.stdout)
|
||||||
|
duration = float(payload["format"]["duration"])
|
||||||
|
except (KeyError, TypeError, ValueError, json.JSONDecodeError):
|
||||||
|
LOG.warning("Ignoring media with incomplete metadata: %s", source)
|
||||||
|
return None
|
||||||
|
|
||||||
|
video_count = 0
|
||||||
|
audio_count = 0
|
||||||
|
attached_picture_stream: int | None = None
|
||||||
|
for stream in payload.get("streams", []):
|
||||||
|
codec_type = stream.get("codec_type")
|
||||||
|
codec_name = stream.get("codec_name")
|
||||||
|
attached_picture = stream.get("disposition", {}).get("attached_pic") == 1
|
||||||
|
if codec_type == "video" and attached_picture:
|
||||||
|
attached_picture_stream = stream.get("index")
|
||||||
|
elif codec_type == "video" and codec_name == "h264":
|
||||||
|
video_count += 1
|
||||||
|
elif codec_type == "audio" and codec_name == "aac":
|
||||||
|
audio_count += 1
|
||||||
|
else:
|
||||||
|
LOG.warning("Ignoring unsupported media %s", source)
|
||||||
|
return None
|
||||||
|
|
||||||
|
if video_count != 1 or audio_count != 1 or duration < 0:
|
||||||
|
LOG.warning("Ignoring unsupported media %s", source)
|
||||||
|
return None
|
||||||
|
return ProbeResult(duration=duration, attached_picture_stream=attached_picture_stream)
|
||||||
|
|
||||||
|
|
||||||
|
def playlist_url(relative_path: Path) -> str:
|
||||||
|
encoded_path = "/".join(quote(part, safe="") for part in relative_path.parts)
|
||||||
|
return f"/hls/{encoded_path}/master.m3u8"
|
||||||
|
|
||||||
|
|
||||||
|
def cache_key(relative_path: Path, stat_result: os.stat_result) -> str:
|
||||||
|
source_fingerprint = f"{relative_path.as_posix()}:{stat_result.st_size}:{stat_result.st_mtime_ns}"
|
||||||
|
return hashlib.sha256(source_fingerprint.encode("utf-8")).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
class Catalog:
|
||||||
|
def __init__(
|
||||||
|
self,
|
||||||
|
media_dir: Path,
|
||||||
|
cache_dir: Path,
|
||||||
|
refresh_interval: float = DEFAULT_REFRESH_INTERVAL_SECONDS,
|
||||||
|
auto_refresh: bool = True,
|
||||||
|
) -> None:
|
||||||
|
self.media_dir = media_dir.resolve()
|
||||||
|
self.cache_dir = cache_dir.resolve()
|
||||||
|
self.releases_dir = self.cache_dir / "releases"
|
||||||
|
self.current_link = self.cache_dir / "current"
|
||||||
|
self.releases_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
self._state_lock = threading.Lock()
|
||||||
|
self._refresh_event = threading.Event()
|
||||||
|
self._stop_event = threading.Event()
|
||||||
|
self._build_active = False
|
||||||
|
self._refresh_pending = False
|
||||||
|
self._current_release: Path | None = None
|
||||||
|
self._media_payload: bytes | None = None
|
||||||
|
self._status: dict[str, object] = {
|
||||||
|
"state": "building",
|
||||||
|
"lastSuccessAt": None,
|
||||||
|
"validFiles": 0,
|
||||||
|
"skippedFiles": 0,
|
||||||
|
"lastError": None,
|
||||||
|
}
|
||||||
|
self._load_current_snapshot()
|
||||||
|
self._refresh_interval = refresh_interval
|
||||||
|
self._worker: threading.Thread | None = None
|
||||||
|
if auto_refresh:
|
||||||
|
self._worker = threading.Thread(target=self._worker_loop, name="catalog-refresh", daemon=True)
|
||||||
|
self._worker.start()
|
||||||
|
self.request_rescan()
|
||||||
|
|
||||||
|
def ready(self) -> bool:
|
||||||
|
with self._state_lock:
|
||||||
|
return self._media_payload is not None and self._current_release is not None
|
||||||
|
|
||||||
|
def media_payload(self) -> bytes | None:
|
||||||
|
with self._state_lock:
|
||||||
|
return self._media_payload
|
||||||
|
|
||||||
|
def cover_payload(self, cover_name: str) -> bytes | None:
|
||||||
|
if not COVER_NAME.fullmatch(cover_name):
|
||||||
|
return None
|
||||||
|
with self._state_lock:
|
||||||
|
release = self._current_release
|
||||||
|
if release is None:
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
return (release / "covers" / cover_name).read_bytes()
|
||||||
|
except OSError:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def status(self) -> dict[str, object]:
|
||||||
|
with self._state_lock:
|
||||||
|
return {"ready": self._media_payload is not None, **self._status}
|
||||||
|
|
||||||
|
def request_rescan(self) -> dict[str, object]:
|
||||||
|
with self._state_lock:
|
||||||
|
if not self._build_active and not self._refresh_pending:
|
||||||
|
self._refresh_pending = True
|
||||||
|
self._status["state"] = "building"
|
||||||
|
self._status["lastError"] = None
|
||||||
|
self._refresh_event.set()
|
||||||
|
return {"ready": self._media_payload is not None, **self._status}
|
||||||
|
|
||||||
|
def refresh_now(self) -> None:
|
||||||
|
"""Synchronously build a snapshot. Intended for tests and worker use."""
|
||||||
|
with self._state_lock:
|
||||||
|
self._build_active = True
|
||||||
|
self._refresh_pending = False
|
||||||
|
self._status["state"] = "building"
|
||||||
|
self._status["lastError"] = None
|
||||||
|
try:
|
||||||
|
release, payload, metadata = self._build_snapshot()
|
||||||
|
self._publish_snapshot(release, payload, metadata)
|
||||||
|
except Exception as error: # Keep the previous release available on a failed build.
|
||||||
|
LOG.exception("Catalog refresh failed")
|
||||||
|
with self._state_lock:
|
||||||
|
self._status["state"] = "ready" if self._media_payload is not None else "failed"
|
||||||
|
self._status["lastError"] = str(error)
|
||||||
|
finally:
|
||||||
|
with self._state_lock:
|
||||||
|
self._build_active = False
|
||||||
|
|
||||||
|
def _worker_loop(self) -> None:
|
||||||
|
while not self._stop_event.is_set():
|
||||||
|
requested = self._refresh_event.wait(self._refresh_interval)
|
||||||
|
if self._stop_event.is_set():
|
||||||
|
return
|
||||||
|
self._refresh_event.clear()
|
||||||
|
with self._state_lock:
|
||||||
|
if not requested and not self._refresh_pending:
|
||||||
|
self._refresh_pending = True
|
||||||
|
self._status["state"] = "building"
|
||||||
|
if self._build_active or not self._refresh_pending:
|
||||||
|
continue
|
||||||
|
self.refresh_now()
|
||||||
|
|
||||||
|
def _build_snapshot(self) -> tuple[Path, bytes, dict[str, object]]:
|
||||||
|
release_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex}"
|
||||||
|
staging = self.releases_dir / f".staging-{release_id}"
|
||||||
|
release = self.releases_dir / release_id
|
||||||
|
covers_dir = staging / "covers"
|
||||||
|
covers_dir.mkdir(parents=True)
|
||||||
|
valid_files = 0
|
||||||
|
skipped_files = 0
|
||||||
|
items: list[dict[str, str]] = []
|
||||||
|
|
||||||
|
try:
|
||||||
|
for source in self._media_files():
|
||||||
|
try:
|
||||||
|
relative_path = source.relative_to(self.media_dir)
|
||||||
|
before = source.stat()
|
||||||
|
except OSError:
|
||||||
|
skipped_files += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
probe = inspect_media(source)
|
||||||
|
if probe is None:
|
||||||
|
skipped_files += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
cover_name = f"{cache_key(relative_path, before)}.jpg"
|
||||||
|
if not self._generate_cover(source, covers_dir / cover_name, probe):
|
||||||
|
skipped_files += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
try:
|
||||||
|
after = source.stat()
|
||||||
|
except OSError:
|
||||||
|
skipped_files += 1
|
||||||
|
continue
|
||||||
|
if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
|
||||||
|
LOG.warning("Ignoring media changed while processing: %s", source)
|
||||||
|
(covers_dir / cover_name).unlink(missing_ok=True)
|
||||||
|
skipped_files += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
valid_files += 1
|
||||||
|
items.append(
|
||||||
|
{
|
||||||
|
"path": relative_path.as_posix(),
|
||||||
|
"name": source.stem,
|
||||||
|
"playlistUrl": playlist_url(relative_path),
|
||||||
|
"coverUrl": f"/api/media/covers/{cover_name}",
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
items.sort(key=lambda item: (item["name"].casefold(), item["path"].casefold()))
|
||||||
|
payload = json.dumps(items, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||||
|
generated_at = datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||||
|
metadata: dict[str, object] = {
|
||||||
|
"generatedAt": generated_at,
|
||||||
|
"validFiles": valid_files,
|
||||||
|
"skippedFiles": skipped_files,
|
||||||
|
}
|
||||||
|
(staging / "media.json").write_bytes(payload)
|
||||||
|
(staging / "metadata.json").write_text(json.dumps(metadata, separators=(",", ":")), encoding="utf-8")
|
||||||
|
staging.replace(release)
|
||||||
|
return release, payload, metadata
|
||||||
|
except Exception:
|
||||||
|
shutil.rmtree(staging, ignore_errors=True)
|
||||||
|
raise
|
||||||
|
|
||||||
|
def _publish_snapshot(self, release: Path, payload: bytes, metadata: dict[str, object]) -> None:
|
||||||
|
pending_link = self.cache_dir / f".current-{uuid.uuid4().hex}"
|
||||||
|
os.symlink(str(release.relative_to(self.cache_dir)), pending_link)
|
||||||
|
try:
|
||||||
|
os.replace(pending_link, self.current_link)
|
||||||
|
finally:
|
||||||
|
pending_link.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
with self._state_lock:
|
||||||
|
self._current_release = release
|
||||||
|
self._media_payload = payload
|
||||||
|
self._status.update(
|
||||||
|
{
|
||||||
|
"state": "ready",
|
||||||
|
"lastSuccessAt": metadata["generatedAt"],
|
||||||
|
"validFiles": metadata["validFiles"],
|
||||||
|
"skippedFiles": metadata["skippedFiles"],
|
||||||
|
"lastError": None,
|
||||||
|
}
|
||||||
|
)
|
||||||
|
self._cleanup_old_releases(release)
|
||||||
|
|
||||||
|
def _load_current_snapshot(self) -> None:
|
||||||
|
try:
|
||||||
|
release = self.current_link.resolve(strict=True)
|
||||||
|
payload = (release / "media.json").read_bytes()
|
||||||
|
items = json.loads(payload)
|
||||||
|
metadata = json.loads((release / "metadata.json").read_text(encoding="utf-8"))
|
||||||
|
if not isinstance(items, list) or not isinstance(metadata, dict):
|
||||||
|
raise ValueError("invalid snapshot")
|
||||||
|
except (OSError, ValueError, json.JSONDecodeError) as error:
|
||||||
|
if self.current_link.exists() or self.current_link.is_symlink():
|
||||||
|
LOG.warning("Ignoring invalid catalog snapshot: %s", error)
|
||||||
|
return
|
||||||
|
|
||||||
|
self._current_release = release
|
||||||
|
self._media_payload = payload
|
||||||
|
self._status.update(
|
||||||
|
{
|
||||||
|
"state": "ready",
|
||||||
|
"lastSuccessAt": metadata.get("generatedAt"),
|
||||||
|
"validFiles": metadata.get("validFiles", 0),
|
||||||
|
"skippedFiles": metadata.get("skippedFiles", 0),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def _cleanup_old_releases(self, active_release: Path) -> None:
|
||||||
|
releases = sorted(
|
||||||
|
(path for path in self.releases_dir.iterdir() if path.is_dir() and not path.name.startswith(".staging-")),
|
||||||
|
key=lambda path: path.stat().st_mtime_ns,
|
||||||
|
reverse=True,
|
||||||
|
)
|
||||||
|
keep = {active_release, *releases[:2]}
|
||||||
|
for release in releases:
|
||||||
|
if release not in keep:
|
||||||
|
shutil.rmtree(release, ignore_errors=True)
|
||||||
|
|
||||||
|
def _generate_cover(self, source: Path, cover_path: Path, probe: ProbeResult) -> bool:
|
||||||
|
temporary_path = cover_path.with_name(f".{cover_path.stem}.{os.getpid()}.jpg")
|
||||||
|
if probe.attached_picture_stream is not None:
|
||||||
|
command = [
|
||||||
|
"ffmpeg",
|
||||||
|
"-nostdin",
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-i",
|
||||||
|
str(source),
|
||||||
|
"-map",
|
||||||
|
f"0:{probe.attached_picture_stream}",
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-vf",
|
||||||
|
"scale=480:-2",
|
||||||
|
"-q:v",
|
||||||
|
"2",
|
||||||
|
"-y",
|
||||||
|
str(temporary_path),
|
||||||
|
]
|
||||||
|
else:
|
||||||
|
timestamp = min(10.0, probe.duration * 0.1)
|
||||||
|
command = [
|
||||||
|
"ffmpeg",
|
||||||
|
"-nostdin",
|
||||||
|
"-v",
|
||||||
|
"error",
|
||||||
|
"-ss",
|
||||||
|
f"{timestamp:.3f}",
|
||||||
|
"-i",
|
||||||
|
str(source),
|
||||||
|
"-frames:v",
|
||||||
|
"1",
|
||||||
|
"-vf",
|
||||||
|
"scale=480:-2",
|
||||||
|
"-q:v",
|
||||||
|
"2",
|
||||||
|
"-y",
|
||||||
|
str(temporary_path),
|
||||||
|
]
|
||||||
|
try:
|
||||||
|
result = subprocess.run(command, capture_output=True, check=False, text=True, timeout=60)
|
||||||
|
if result.returncode != 0 or not temporary_path.is_file() or temporary_path.stat().st_size == 0:
|
||||||
|
LOG.warning("Unable to generate cover for %s: %s", source, result.stderr.strip())
|
||||||
|
return False
|
||||||
|
temporary_path.replace(cover_path)
|
||||||
|
return True
|
||||||
|
except (OSError, subprocess.TimeoutExpired) as error:
|
||||||
|
LOG.warning("Unable to generate cover for %s: %s", source, error)
|
||||||
|
return False
|
||||||
|
finally:
|
||||||
|
temporary_path.unlink(missing_ok=True)
|
||||||
|
|
||||||
|
def _media_files(self) -> list[Path]:
|
||||||
|
files: list[Path] = []
|
||||||
|
for root, directories, filenames in os.walk(self.media_dir, followlinks=False):
|
||||||
|
directories[:] = [name for name in directories if not (Path(root) / name).is_symlink()]
|
||||||
|
for filename in filenames:
|
||||||
|
source = Path(root) / filename
|
||||||
|
if source.suffix.lower() == ".mp4" and source.is_file() and not source.is_symlink():
|
||||||
|
files.append(source)
|
||||||
|
return sorted(files, key=lambda path: path.relative_to(self.media_dir).as_posix().casefold())
|
||||||
|
|
||||||
|
|
||||||
|
class CatalogRequestHandler(BaseHTTPRequestHandler):
|
||||||
|
server: "CatalogHTTPServer"
|
||||||
|
|
||||||
|
def do_GET(self) -> None: # noqa: N802
|
||||||
|
request = urlsplit(self.path)
|
||||||
|
if request.path == "/healthz":
|
||||||
|
if self.server.catalog.ready():
|
||||||
|
self._send_bytes(HTTPStatus.OK, b"ok\n", "text/plain; charset=utf-8")
|
||||||
|
else:
|
||||||
|
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"error": "catalog is building"})
|
||||||
|
elif request.path == "/api/media/status":
|
||||||
|
self._send_json(HTTPStatus.OK, self.server.catalog.status())
|
||||||
|
elif request.path == "/api/media":
|
||||||
|
payload = self.server.catalog.media_payload()
|
||||||
|
if payload is None:
|
||||||
|
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"error": "catalog is building"})
|
||||||
|
else:
|
||||||
|
self._send_bytes(HTTPStatus.OK, payload, "application/json; charset=utf-8")
|
||||||
|
elif request.path.startswith("/api/media/covers/"):
|
||||||
|
if not self.server.catalog.ready():
|
||||||
|
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"error": "catalog is building"})
|
||||||
|
else:
|
||||||
|
self._send_cover(request.path.rsplit("/", 1)[-1])
|
||||||
|
else:
|
||||||
|
self.send_error(HTTPStatus.NOT_FOUND)
|
||||||
|
|
||||||
|
def do_HEAD(self) -> None: # noqa: N802
|
||||||
|
self.do_GET()
|
||||||
|
|
||||||
|
def do_POST(self) -> None: # noqa: N802
|
||||||
|
if urlsplit(self.path).path != "/api/media/rescan":
|
||||||
|
self.send_error(HTTPStatus.NOT_FOUND)
|
||||||
|
return
|
||||||
|
self._send_json(HTTPStatus.ACCEPTED, self.server.catalog.request_rescan())
|
||||||
|
|
||||||
|
def do_OPTIONS(self) -> None: # noqa: N802
|
||||||
|
self.send_response(HTTPStatus.NO_CONTENT)
|
||||||
|
self.send_header("Allow", "GET, HEAD, POST, OPTIONS")
|
||||||
|
self.end_headers()
|
||||||
|
|
||||||
|
def _send_cover(self, cover_name: str) -> None:
|
||||||
|
payload = self.server.catalog.cover_payload(cover_name)
|
||||||
|
if payload is None:
|
||||||
|
self.send_error(HTTPStatus.NOT_FOUND)
|
||||||
|
return
|
||||||
|
self._send_bytes(HTTPStatus.OK, payload, "image/jpeg")
|
||||||
|
|
||||||
|
def _send_json(self, status: HTTPStatus, payload: object) -> None:
|
||||||
|
self._send_bytes(
|
||||||
|
status,
|
||||||
|
json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
|
||||||
|
"application/json; charset=utf-8",
|
||||||
|
)
|
||||||
|
|
||||||
|
def _send_bytes(self, status: HTTPStatus, body: bytes, content_type: str) -> None:
|
||||||
|
self.send_response(status)
|
||||||
|
self.send_header("Content-Type", content_type)
|
||||||
|
self.send_header("Content-Length", str(len(body)))
|
||||||
|
self.send_header("Cache-Control", "no-store")
|
||||||
|
self.end_headers()
|
||||||
|
if self.command != "HEAD":
|
||||||
|
self.wfile.write(body)
|
||||||
|
|
||||||
|
def log_message(self, format_string: str, *args: object) -> None:
|
||||||
|
LOG.info("%s - %s", self.address_string(), format_string % args)
|
||||||
|
|
||||||
|
|
||||||
|
class CatalogHTTPServer(ThreadingHTTPServer):
|
||||||
|
daemon_threads = True
|
||||||
|
|
||||||
|
def __init__(self, address: tuple[str, int], catalog: Catalog) -> None:
|
||||||
|
super().__init__(address, CatalogRequestHandler)
|
||||||
|
self.catalog = catalog
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> None:
|
||||||
|
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s")
|
||||||
|
media_dir = Path(os.environ.get("MEDIA_DIR", "/media"))
|
||||||
|
cache_dir = Path(os.environ.get("CATALOG_CACHE_DIR", "/cache"))
|
||||||
|
refresh_interval = float(os.environ.get("CATALOG_REFRESH_INTERVAL_SECONDS", str(DEFAULT_REFRESH_INTERVAL_SECONDS)))
|
||||||
|
port = int(os.environ.get("CATALOG_PORT", "8081"))
|
||||||
|
if not media_dir.is_dir():
|
||||||
|
raise SystemExit(f"Media directory does not exist: {media_dir}")
|
||||||
|
if refresh_interval <= 0:
|
||||||
|
raise SystemExit("CATALOG_REFRESH_INTERVAL_SECONDS must be positive")
|
||||||
|
CatalogHTTPServer(("0.0.0.0", port), Catalog(media_dir, cache_dir, refresh_interval)).serve_forever()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
@@ -0,0 +1,69 @@
|
|||||||
|
services:
|
||||||
|
vod:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
target: vod
|
||||||
|
args:
|
||||||
|
NGINX_VERSION: "${NGINX_VERSION:-1.30.5}"
|
||||||
|
KALTURA_VOD_REF: "${KALTURA_VOD_REF:-26f06877b0f2a2336e59cda93a3de18d7b23a3e2}"
|
||||||
|
image: "${IMAGE_NAME:-nginx-hls-vod}:local"
|
||||||
|
environment:
|
||||||
|
PLAYER_ORIGIN: "${PLAYER_ORIGIN:?Set PLAYER_ORIGIN in .env}"
|
||||||
|
depends_on:
|
||||||
|
catalog:
|
||||||
|
condition: service_healthy
|
||||||
|
ports:
|
||||||
|
- "${HLS_BIND_ADDRESS:-0.0.0.0}:${HLS_PORT:-8124}:8080"
|
||||||
|
volumes:
|
||||||
|
- type: bind
|
||||||
|
source: "${MEDIA_DIR:?Set MEDIA_DIR in .env}"
|
||||||
|
target: /media
|
||||||
|
read_only: true
|
||||||
|
bind:
|
||||||
|
create_host_path: false
|
||||||
|
restart: unless-stopped
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp:rw,nosuid,nodev,noexec,size=64m
|
||||||
|
cap_drop:
|
||||||
|
- ALL
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
pids_limit: 128
|
||||||
|
mem_limit: 512m
|
||||||
|
cpus: 2.0
|
||||||
|
|
||||||
|
catalog:
|
||||||
|
build:
|
||||||
|
context: .
|
||||||
|
target: catalog
|
||||||
|
image: "${CATALOG_IMAGE:-nginx-hls-catalog}:local"
|
||||||
|
user: "0:0"
|
||||||
|
environment:
|
||||||
|
MEDIA_DIR: /media
|
||||||
|
CATALOG_CACHE_DIR: /cache
|
||||||
|
CATALOG_REFRESH_INTERVAL_SECONDS: "${CATALOG_REFRESH_INTERVAL_SECONDS:-21600}"
|
||||||
|
CATALOG_PORT: "8081"
|
||||||
|
volumes:
|
||||||
|
- type: bind
|
||||||
|
source: "${MEDIA_DIR:?Set MEDIA_DIR in .env}"
|
||||||
|
target: /media
|
||||||
|
read_only: true
|
||||||
|
bind:
|
||||||
|
create_host_path: false
|
||||||
|
- type: bind
|
||||||
|
source: "${CATALOG_CACHE_DIR:?Set CATALOG_CACHE_DIR in .env}"
|
||||||
|
target: /cache
|
||||||
|
bind:
|
||||||
|
create_host_path: false
|
||||||
|
restart: unless-stopped
|
||||||
|
read_only: true
|
||||||
|
tmpfs:
|
||||||
|
- /tmp:rw,nosuid,nodev,noexec,size=64m
|
||||||
|
cap_drop:
|
||||||
|
- ALL
|
||||||
|
security_opt:
|
||||||
|
- no-new-privileges:true
|
||||||
|
pids_limit: 128
|
||||||
|
mem_limit: 512m
|
||||||
|
cpus: 2.0
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
worker_processes auto;
|
||||||
|
error_log /dev/stderr warn;
|
||||||
|
pid /tmp/nginx.pid;
|
||||||
|
|
||||||
|
events {
|
||||||
|
worker_connections 1024;
|
||||||
|
}
|
||||||
|
|
||||||
|
thread_pool vod_io threads=2 max_queue=65536;
|
||||||
|
|
||||||
|
http {
|
||||||
|
include /etc/nginx/mime.types;
|
||||||
|
default_type application/octet-stream;
|
||||||
|
|
||||||
|
log_format vod '$remote_addr - $remote_user [$time_local] "$request" '
|
||||||
|
'$status $body_bytes_sent "$http_referer" '
|
||||||
|
'"$http_user_agent" rt=$request_time';
|
||||||
|
access_log /dev/stdout vod;
|
||||||
|
|
||||||
|
sendfile on;
|
||||||
|
tcp_nopush on;
|
||||||
|
keepalive_timeout 65;
|
||||||
|
server_tokens off;
|
||||||
|
|
||||||
|
aio threads=vod_io;
|
||||||
|
|
||||||
|
upstream catalog_backend {
|
||||||
|
server catalog:8081;
|
||||||
|
}
|
||||||
|
|
||||||
|
server {
|
||||||
|
listen 8080;
|
||||||
|
server_name _;
|
||||||
|
autoindex off;
|
||||||
|
|
||||||
|
vod_mode local;
|
||||||
|
vod_metadata_cache metadata_cache 64m;
|
||||||
|
vod_response_cache response_cache 16m;
|
||||||
|
vod_open_file_thread_pool vod_io;
|
||||||
|
|
||||||
|
open_file_cache max=1000 inactive=30s;
|
||||||
|
open_file_cache_valid 30s;
|
||||||
|
open_file_cache_min_uses 1;
|
||||||
|
open_file_cache_errors on;
|
||||||
|
|
||||||
|
location = /healthz {
|
||||||
|
access_log off;
|
||||||
|
default_type text/plain;
|
||||||
|
return 200 "ok\n";
|
||||||
|
}
|
||||||
|
|
||||||
|
location ^~ /hls/ {
|
||||||
|
alias /media/;
|
||||||
|
vod hls;
|
||||||
|
vod_segment_duration 6000;
|
||||||
|
vod_align_segments_to_key_frames on;
|
||||||
|
vod_manifest_segment_durations_mode accurate;
|
||||||
|
|
||||||
|
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||||
|
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||||
|
add_header Access-Control-Expose-Headers "Accept-Ranges, Content-Length, Content-Range, Content-Type" always;
|
||||||
|
add_header Cache-Control "no-store" always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
|
||||||
|
if ($request_method = OPTIONS) {
|
||||||
|
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||||
|
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||||
|
add_header Access-Control-Max-Age 600 always;
|
||||||
|
add_header Cache-Control "no-store" always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
return 204;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($request_method !~ ^(GET|HEAD)$) {
|
||||||
|
return 405;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
location = /api/media {
|
||||||
|
proxy_pass http://catalog_backend;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||||
|
proxy_set_header X-Forwarded-Proto $scheme;
|
||||||
|
proxy_hide_header Cache-Control;
|
||||||
|
|
||||||
|
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||||
|
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||||
|
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
|
||||||
|
add_header Cache-Control "no-store" always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
|
||||||
|
if ($request_method = OPTIONS) {
|
||||||
|
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||||
|
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||||
|
add_header Access-Control-Max-Age 600 always;
|
||||||
|
add_header Cache-Control "no-store" always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
return 204;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($request_method !~ ^(GET|HEAD)$) {
|
||||||
|
return 405;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
location = /api/media/status {
|
||||||
|
proxy_pass http://catalog_backend;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_hide_header Cache-Control;
|
||||||
|
|
||||||
|
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||||
|
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||||
|
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
|
||||||
|
add_header Cache-Control "no-store" always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
|
||||||
|
if ($request_method = OPTIONS) {
|
||||||
|
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||||
|
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||||
|
add_header Access-Control-Max-Age 600 always;
|
||||||
|
add_header Cache-Control "no-store" always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
return 204;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($request_method !~ ^(GET|HEAD)$) {
|
||||||
|
return 405;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
location = /api/media/rescan {
|
||||||
|
proxy_pass http://catalog_backend;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_hide_header Cache-Control;
|
||||||
|
|
||||||
|
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||||
|
add_header Access-Control-Allow-Methods "POST, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||||
|
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
|
||||||
|
add_header Cache-Control "no-store" always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
|
||||||
|
if ($request_method = OPTIONS) {
|
||||||
|
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||||
|
add_header Access-Control-Allow-Methods "POST, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||||
|
add_header Access-Control-Max-Age 600 always;
|
||||||
|
add_header Cache-Control "no-store" always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
return 204;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($request_method != POST) {
|
||||||
|
return 405;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
location ^~ /api/media/covers/ {
|
||||||
|
proxy_pass http://catalog_backend;
|
||||||
|
proxy_set_header Host $host;
|
||||||
|
proxy_hide_header Cache-Control;
|
||||||
|
|
||||||
|
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||||
|
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||||
|
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
|
||||||
|
add_header Cache-Control "no-store" always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
|
||||||
|
if ($request_method = OPTIONS) {
|
||||||
|
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||||
|
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||||
|
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||||
|
add_header Access-Control-Max-Age 600 always;
|
||||||
|
add_header Cache-Control "no-store" always;
|
||||||
|
add_header Vary "Origin" always;
|
||||||
|
return 204;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($request_method !~ ^(GET|HEAD)$) {
|
||||||
|
return 405;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
location / {
|
||||||
|
return 404;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
Executable
+17
@@ -0,0 +1,17 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
set -eu
|
||||||
|
|
||||||
|
: "${PLAYER_ORIGIN:?PLAYER_ORIGIN must be set}"
|
||||||
|
|
||||||
|
if ! printf '%s' "$PLAYER_ORIGIN" | grep -Eq '^https?://[A-Za-z0-9._:-]+$'; then
|
||||||
|
echo "PLAYER_ORIGIN must be an origin such as http://192.168.1.20:3000" >&2
|
||||||
|
exit 64
|
||||||
|
fi
|
||||||
|
|
||||||
|
envsubst '${PLAYER_ORIGIN}' \
|
||||||
|
< /etc/nginx/templates/nginx.conf.template \
|
||||||
|
> /tmp/nginx.conf
|
||||||
|
|
||||||
|
nginx -t -c /tmp/nginx.conf
|
||||||
|
exec nginx -c /tmp/nginx.conf -g 'daemon off;'
|
||||||
|
|
||||||
Executable
+6
@@ -0,0 +1,6 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
echo "Restarting VOD after an already-completed atomic media update."
|
||||||
|
docker compose restart vod
|
||||||
|
|
||||||
Executable
+52
@@ -0,0 +1,52 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
: "${PLAYER_ORIGIN:?Set PLAYER_ORIGIN before running this test}"
|
||||||
|
|
||||||
|
base_url=${HLS_BASE_URL:-http://127.0.0.1:8124}
|
||||||
|
media_path=${TEST_MEDIA_PATH:-aa/1170193193-1-192.mp4}
|
||||||
|
tmpdir=$(mktemp -d)
|
||||||
|
trap 'rm -rf "$tmpdir"' EXIT
|
||||||
|
|
||||||
|
resolve_url() {
|
||||||
|
local base=$1
|
||||||
|
local reference=$2
|
||||||
|
|
||||||
|
case "$reference" in
|
||||||
|
http://*|https://*) printf '%s\n' "$reference" ;;
|
||||||
|
/*) printf '%s%s\n' "$(printf '%s\n' "$base" | sed -E 's#(https?://[^/]+).*#\1#')" "$reference" ;;
|
||||||
|
*) printf '%s/%s\n' "${base%/*}" "$reference" ;;
|
||||||
|
esac
|
||||||
|
}
|
||||||
|
|
||||||
|
master_url="${base_url%/}/hls/${media_path}/master.m3u8"
|
||||||
|
master_headers="$tmpdir/master.headers"
|
||||||
|
master_playlist="$tmpdir/master.m3u8"
|
||||||
|
|
||||||
|
curl --fail --silent --show-error --dump-header "$master_headers" --output "$master_playlist" "$master_url"
|
||||||
|
tr -d '\r' < "$master_headers" | grep -Fxi "Access-Control-Allow-Origin: $PLAYER_ORIGIN" >/dev/null
|
||||||
|
tr -d '\r' < "$master_headers" | grep -Fxi 'Cache-Control: no-store' >/dev/null
|
||||||
|
grep -Fqx '#EXTM3U' "$master_playlist" >/dev/null
|
||||||
|
|
||||||
|
child_reference=$(awk 'found { print; exit } /^#EXT-X-STREAM-INF/ { found=1 }' "$master_playlist")
|
||||||
|
if [[ -z $child_reference ]]; then
|
||||||
|
child_url=$master_url
|
||||||
|
else
|
||||||
|
child_url=$(resolve_url "$master_url" "$child_reference")
|
||||||
|
fi
|
||||||
|
|
||||||
|
child_playlist="$tmpdir/index.m3u8"
|
||||||
|
curl --fail --silent --show-error --output "$child_playlist" "$child_url"
|
||||||
|
grep -Fqx '#EXTM3U' "$child_playlist" >/dev/null
|
||||||
|
|
||||||
|
segment_reference=$(awk '!/^#/ && NF { print; exit }' "$child_playlist")
|
||||||
|
if [[ -z $segment_reference ]]; then
|
||||||
|
echo "No HLS segment was found in $child_url" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
segment_url=$(resolve_url "$child_url" "$segment_reference")
|
||||||
|
curl --fail --silent --show-error --output /dev/null "$segment_url"
|
||||||
|
|
||||||
|
printf 'HLS smoke test passed: %s\n' "$master_url"
|
||||||
|
|
||||||
Executable
+100
@@ -0,0 +1,100 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
media_root=${1:-data}
|
||||||
|
|
||||||
|
if ! command -v ffprobe >/dev/null 2>&1; then
|
||||||
|
echo "ffprobe is required to validate media." >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ! -d $media_root ]]; then
|
||||||
|
printf 'Media directory does not exist: %s\n' "$media_root" >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
valid_count=0
|
||||||
|
invalid_count=0
|
||||||
|
|
||||||
|
validate_file() {
|
||||||
|
local file=$1
|
||||||
|
local format_name streams codec type attached_pic
|
||||||
|
local video_count=0
|
||||||
|
local audio_count=0
|
||||||
|
local invalid_reason=
|
||||||
|
|
||||||
|
if ! format_name=$(ffprobe -v error -show_entries format=format_name \
|
||||||
|
-of default=noprint_wrappers=1:nokey=1 "$file"); then
|
||||||
|
printf 'INVALID: %q: unreadable or corrupt MP4\n' "$file" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [[ ,$format_name, != *,mp4,* ]]; then
|
||||||
|
printf 'INVALID: %q: expected an MP4 container, got %s\n' "$file" "$format_name" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ! streams=$(ffprobe -v error \
|
||||||
|
-show_entries stream=codec_name,codec_type:stream_disposition=attached_pic \
|
||||||
|
-of csv=p=0 "$file"); then
|
||||||
|
printf 'INVALID: %q: unable to inspect streams\n' "$file" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
while IFS=, read -r codec type attached_pic; do
|
||||||
|
case "$type" in
|
||||||
|
video)
|
||||||
|
if [[ $attached_pic == 1 ]]; then
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
if [[ $codec != h264 ]]; then
|
||||||
|
invalid_reason="unsupported video codec $codec"
|
||||||
|
fi
|
||||||
|
((video_count += 1))
|
||||||
|
;;
|
||||||
|
audio)
|
||||||
|
if [[ $codec != aac ]]; then
|
||||||
|
invalid_reason="unsupported audio codec $codec"
|
||||||
|
fi
|
||||||
|
((audio_count += 1))
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
invalid_reason="unsupported $type track ($codec)"
|
||||||
|
;;
|
||||||
|
esac
|
||||||
|
done <<< "$streams"
|
||||||
|
|
||||||
|
if [[ -n $invalid_reason ]]; then
|
||||||
|
printf 'INVALID: %q: %s\n' "$file" "$invalid_reason" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ((video_count != 1 || audio_count != 1)); then
|
||||||
|
printf 'INVALID: %q: expected one H.264 video and one AAC audio track; found %d video and %d audio\n' \
|
||||||
|
"$file" "$video_count" "$audio_count" >&2
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'VALID: %s\n' "$file"
|
||||||
|
}
|
||||||
|
|
||||||
|
while IFS= read -r -d '' file; do
|
||||||
|
if validate_file "$file"; then
|
||||||
|
((valid_count += 1))
|
||||||
|
else
|
||||||
|
((invalid_count += 1))
|
||||||
|
fi
|
||||||
|
done < <(find "$media_root" -type f -iname '*.mp4' -print0)
|
||||||
|
|
||||||
|
if ((valid_count == 0)); then
|
||||||
|
echo "No valid MP4 files were found." >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if ((invalid_count > 0)); then
|
||||||
|
printf 'Validation failed: %d valid, %d invalid file(s).\n' "$valid_count" "$invalid_count" >&2
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf 'Validation passed: %d MP4 file(s).\n' "$valid_count"
|
||||||
|
|
||||||
@@ -0,0 +1,153 @@
|
|||||||
|
import json
|
||||||
|
import subprocess
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from pathlib import Path
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from catalog.app import Catalog, ProbeResult, cache_key, inspect_media, playlist_url
|
||||||
|
|
||||||
|
|
||||||
|
class CatalogTests(unittest.TestCase):
|
||||||
|
def test_playlist_url_encodes_each_path_segment(self) -> None:
|
||||||
|
self.assertEqual(
|
||||||
|
playlist_url(Path("中文 media") / "clip #1.mp4"),
|
||||||
|
"/hls/%E4%B8%AD%E6%96%87%20media/clip%20%231.mp4/master.m3u8",
|
||||||
|
)
|
||||||
|
|
||||||
|
def test_cache_key_changes_when_media_changes(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||||
|
source = Path(temporary_directory) / "clip.mp4"
|
||||||
|
source.write_bytes(b"a")
|
||||||
|
first = cache_key(Path("clip.mp4"), source.stat())
|
||||||
|
source.write_bytes(b"updated")
|
||||||
|
second = cache_key(Path("clip.mp4"), source.stat())
|
||||||
|
self.assertNotEqual(first, second)
|
||||||
|
|
||||||
|
def test_refresh_publishes_validated_list_and_cached_cover(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||||
|
root = Path(temporary_directory)
|
||||||
|
media_dir = root / "media"
|
||||||
|
cache_dir = root / "cache"
|
||||||
|
media_dir.mkdir()
|
||||||
|
source = media_dir / "clip.mp4"
|
||||||
|
source.write_bytes(b"video")
|
||||||
|
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
|
||||||
|
probe_payload = json.dumps(
|
||||||
|
{
|
||||||
|
"format": {"duration": "20"},
|
||||||
|
"streams": [
|
||||||
|
{"index": 0, "codec_type": "video", "codec_name": "h264", "disposition": {"attached_pic": 0}},
|
||||||
|
{"index": 1, "codec_type": "audio", "codec_name": "aac", "disposition": {"attached_pic": 0}},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
def fake_run(command: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]:
|
||||||
|
if command[0] == "ffprobe":
|
||||||
|
return subprocess.CompletedProcess(command, 0, probe_payload, "")
|
||||||
|
Path(command[-1]).write_bytes(b"jpeg")
|
||||||
|
return subprocess.CompletedProcess(command, 0, "", "")
|
||||||
|
|
||||||
|
self.assertFalse(catalog.ready())
|
||||||
|
with patch("catalog.app.subprocess.run", side_effect=fake_run) as run:
|
||||||
|
catalog.refresh_now()
|
||||||
|
|
||||||
|
self.assertTrue(catalog.ready())
|
||||||
|
self.assertTrue((cache_dir / "current").is_symlink())
|
||||||
|
items = json.loads(catalog.media_payload())
|
||||||
|
self.assertEqual(len(items), 1)
|
||||||
|
self.assertEqual(items[0]["path"], "clip.mp4")
|
||||||
|
self.assertEqual(items[0]["playlistUrl"], "/hls/clip.mp4/master.m3u8")
|
||||||
|
cover_name = items[0]["coverUrl"].rsplit("/", 1)[-1]
|
||||||
|
self.assertEqual(catalog.cover_payload(cover_name), b"jpeg")
|
||||||
|
self.assertEqual(run.call_count, 2)
|
||||||
|
|
||||||
|
with patch("catalog.app.subprocess.run") as run:
|
||||||
|
self.assertEqual(json.loads(catalog.media_payload()), items)
|
||||||
|
self.assertEqual(catalog.cover_payload(cover_name), b"jpeg")
|
||||||
|
run.assert_not_called()
|
||||||
|
|
||||||
|
def test_refresh_excludes_invalid_media(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||||
|
root = Path(temporary_directory)
|
||||||
|
media_dir = root / "media"
|
||||||
|
cache_dir = root / "cache"
|
||||||
|
media_dir.mkdir()
|
||||||
|
(media_dir / "valid.mp4").write_bytes(b"valid")
|
||||||
|
(media_dir / "broken.mp4").write_bytes(b"broken")
|
||||||
|
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
|
||||||
|
|
||||||
|
def fake_probe(source: Path) -> ProbeResult | None:
|
||||||
|
return ProbeResult(10.0, None) if source.name == "valid.mp4" else None
|
||||||
|
|
||||||
|
with patch("catalog.app.inspect_media", side_effect=fake_probe), patch.object(
|
||||||
|
Catalog, "_generate_cover", return_value=True
|
||||||
|
):
|
||||||
|
catalog.refresh_now()
|
||||||
|
|
||||||
|
items = json.loads(catalog.media_payload())
|
||||||
|
self.assertEqual([item["path"] for item in items], ["valid.mp4"])
|
||||||
|
status = catalog.status()
|
||||||
|
self.assertEqual(status["validFiles"], 1)
|
||||||
|
self.assertEqual(status["skippedFiles"], 1)
|
||||||
|
|
||||||
|
def test_failed_refresh_keeps_previous_snapshot(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||||
|
root = Path(temporary_directory)
|
||||||
|
media_dir = root / "media"
|
||||||
|
cache_dir = root / "cache"
|
||||||
|
media_dir.mkdir()
|
||||||
|
(media_dir / "clip.mp4").write_bytes(b"video")
|
||||||
|
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
|
||||||
|
|
||||||
|
with patch("catalog.app.inspect_media", return_value=ProbeResult(10.0, None)), patch.object(
|
||||||
|
Catalog, "_generate_cover", return_value=True
|
||||||
|
):
|
||||||
|
catalog.refresh_now()
|
||||||
|
previous_payload = catalog.media_payload()
|
||||||
|
|
||||||
|
with patch.object(Catalog, "_build_snapshot", side_effect=RuntimeError("disk full")):
|
||||||
|
catalog.refresh_now()
|
||||||
|
|
||||||
|
self.assertTrue(catalog.ready())
|
||||||
|
self.assertEqual(catalog.media_payload(), previous_payload)
|
||||||
|
self.assertEqual(catalog.status()["state"], "ready")
|
||||||
|
self.assertEqual(catalog.status()["lastError"], "disk full")
|
||||||
|
|
||||||
|
def test_existing_snapshot_is_loaded_without_subprocesses(self) -> None:
|
||||||
|
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||||
|
root = Path(temporary_directory)
|
||||||
|
media_dir = root / "media"
|
||||||
|
cache_dir = root / "cache"
|
||||||
|
media_dir.mkdir()
|
||||||
|
(media_dir / "clip.mp4").write_bytes(b"video")
|
||||||
|
builder = Catalog(media_dir, cache_dir, auto_refresh=False)
|
||||||
|
with patch("catalog.app.inspect_media", return_value=ProbeResult(10.0, None)), patch.object(
|
||||||
|
Catalog, "_generate_cover", return_value=True
|
||||||
|
):
|
||||||
|
builder.refresh_now()
|
||||||
|
|
||||||
|
with patch("catalog.app.subprocess.run") as run:
|
||||||
|
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
|
||||||
|
payload = catalog.media_payload()
|
||||||
|
run.assert_not_called()
|
||||||
|
self.assertTrue(catalog.ready())
|
||||||
|
self.assertEqual(len(json.loads(payload)), 1)
|
||||||
|
|
||||||
|
def test_inspect_media_rejects_unsupported_streams(self) -> None:
|
||||||
|
payload = {
|
||||||
|
"format": {"duration": "12.5"},
|
||||||
|
"streams": [
|
||||||
|
{"index": 0, "codec_type": "video", "codec_name": "h264", "disposition": {"attached_pic": 0}},
|
||||||
|
{"index": 1, "codec_type": "audio", "codec_name": "aac", "disposition": {"attached_pic": 0}},
|
||||||
|
{"index": 2, "codec_type": "subtitle", "codec_name": "mov_text", "disposition": {"attached_pic": 0}},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
completed = subprocess.CompletedProcess([], 0, json.dumps(payload), "")
|
||||||
|
with patch("catalog.app.subprocess.run", return_value=completed):
|
||||||
|
self.assertIsNone(inspect_media(Path("clip.mp4")))
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
unittest.main()
|
||||||
Reference in New Issue
Block a user