This commit is contained in:
root
2026-09-27 15:55:38 +08:00
commit 1b6f2e6831
14 changed files with 1379 additions and 0 deletions
+7
View File
@@ -0,0 +1,7 @@
.env
.git
.gitignore
data
catalog-cache
README.md
scripts
+19
View File
@@ -0,0 +1,19 @@
# Exact browser origin allowed to request HLS resources. This value is required.
PLAYER_ORIGIN=http://192.168.1.20:3000
# Development uses the repository test library. Production must use an absolute host path.
MEDIA_DIR=./data
# Persistent host directory for catalog JSON snapshots and generated JPEG covers.
# Create it before starting the stack.
CATALOG_CACHE_DIR=./catalog-cache
CATALOG_REFRESH_INTERVAL_SECONDS=21600
# LAN HTTP listener published by Docker Compose.
HLS_BIND_ADDRESS=0.0.0.0
HLS_PORT=8124
# Fixed build inputs. Upgrade these deliberately and re-run the smoke test.
NGINX_VERSION=1.30.5
KALTURA_VOD_REF=26f06877b0f2a2336e59cda93a3de18d7b23a3e2
IMAGE_NAME=nginx-hls-vod
+30
View File
@@ -0,0 +1,30 @@
# Local configuration and runtime catalog cache
.env
.env.*
!.env.example
catalog-cache/
# Python caches, virtual environments, and test coverage
__pycache__/
*.py[cod]
.pytest_cache/
.coverage
coverage.xml
htmlcov/
.venv/
venv/
env/
# Local Docker overrides and editor/OS files
compose.override.yml
compose.override.yaml
docker-compose.override.yml
*.log
.DS_Store
Thumbs.db
.idea/
*.iml
# Test media stays local and is never sent as a Docker build context.
data/*
!data/.gitkeep
+96
View File
@@ -0,0 +1,96 @@
FROM debian:bookworm-slim AS builder
ARG NGINX_VERSION=1.30.5
ARG KALTURA_VOD_REF=26f06877b0f2a2336e59cda93a3de18d7b23a3e2
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
build-essential \
ca-certificates \
curl \
libpcre2-dev \
zlib1g-dev \
&& rm -rf /var/lib/apt/lists/*
RUN mkdir -p /src/nginx-vod-module \
&& curl -fsSL "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" -o /tmp/nginx.tar.gz \
&& curl -fsSL "https://codeload.github.com/kaltura/nginx-vod-module/tar.gz/${KALTURA_VOD_REF}" -o /tmp/nginx-vod-module.tar.gz \
&& tar -xzf /tmp/nginx.tar.gz -C /src \
&& tar -xzf /tmp/nginx-vod-module.tar.gz -C /src/nginx-vod-module --strip-components=1 \
&& rm -f /tmp/nginx.tar.gz /tmp/nginx-vod-module.tar.gz
WORKDIR /src/nginx-${NGINX_VERSION}
RUN ./configure \
--prefix=/usr/share/nginx \
--sbin-path=/usr/sbin/nginx \
--conf-path=/etc/nginx/nginx.conf \
--pid-path=/tmp/nginx.pid \
--error-log-path=/dev/stderr \
--http-log-path=/dev/stdout \
--http-client-body-temp-path=/tmp/client_body \
--http-proxy-temp-path=/tmp/proxy \
--http-fastcgi-temp-path=/tmp/fastcgi \
--http-uwsgi-temp-path=/tmp/uwsgi \
--http-scgi-temp-path=/tmp/scgi \
--with-file-aio \
--with-threads \
--with-http_gzip_static_module \
--with-http_stub_status_module \
--with-cc-opt='-O2' \
--add-module=/src/nginx-vod-module \
&& make -j"$(nproc)" \
&& make install
FROM debian:bookworm-slim AS vod
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ca-certificates \
curl \
gettext-base \
libpcre2-8-0 \
zlib1g \
&& addgroup --system nginx \
&& adduser --system --ingroup nginx --no-create-home --home /nonexistent nginx \
&& mkdir -p /etc/nginx/templates \
&& rm -rf /var/lib/apt/lists/*
COPY --from=builder /usr/sbin/nginx /usr/sbin/nginx
COPY --from=builder /etc/nginx/mime.types /etc/nginx/mime.types
COPY config/nginx.conf.template /etc/nginx/templates/nginx.conf.template
COPY docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh
RUN chmod 0755 /usr/local/bin/docker-entrypoint.sh
USER nginx
EXPOSE 8080
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
CMD curl --fail --silent http://127.0.0.1:8080/healthz || exit 1
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
FROM debian:bookworm-slim AS catalog
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
ffmpeg \
python3 \
&& addgroup --system catalog \
&& adduser --system --ingroup catalog --no-create-home --home /nonexistent catalog \
&& mkdir -p /app /cache \
&& chown catalog:catalog /app /cache \
&& rm -rf /var/lib/apt/lists/*
COPY catalog/app.py /app/app.py
USER catalog
EXPOSE 8081
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
CMD python3 -c "from urllib.request import urlopen; urlopen('http://127.0.0.1:8081/healthz', timeout=3).read()" || exit 1
ENTRYPOINT ["python3", "/app/app.py"]
+159
View File
@@ -0,0 +1,159 @@
# NGINX HLS VOD
Docker Compose deployment for local MP4 to HLS VOD packaging using Kaltura
`nginx-vod-module`. The service maps a media tree read-only into the container
and preserves its hierarchy in HLS URLs.
## Scope
- ARM64 Linux Docker host
- Trusted LAN HTTP service on port `8124`
- One H.264 video track and one AAC audio track per MP4
- One configured browser origin for CORS
- No authentication, TLS, ABR, external subtitles, or multiple audio/video
tracks
An attached-picture video stream such as MP4 cover art is ignored by the media
validator.
## Configuration
Copy the environment template and replace the example player origin:
```sh
cp .env.example .env
```
Required values:
```dotenv
PLAYER_ORIGIN=http://192.168.1.20:3000
MEDIA_DIR=/srv/media
CATALOG_CACHE_DIR=/srv/nginx-hls/catalog-cache
HLS_PORT=8124
```
`MEDIA_DIR=./data` is appropriate only for local development. Production uses
an absolute path on the Docker host. The container user must be able to read
the mounted files and directories. `CATALOG_CACHE_DIR` stores catalog snapshots
and generated covers. Create it before startup, for example
`install -d -m 0700 /srv/nginx-hls/catalog-cache`.
`CATALOG_REFRESH_INTERVAL_SECONDS` defaults to `21600` (6 hours).
The Dockerfile deliberately fixes NGINX `1.30.5` and Kaltura module commit
`26f06877b0f2a2336e59cda93a3de18d7b23a3e2`. Change either only as a deliberate
upgrade, then rebuild and run the smoke test.
## Run
Start the stack after creating the configured cache directory:
```sh
set -a
. ./.env
set +a
install -d -m 0700 "$CATALOG_CACHE_DIR"
docker compose --env-file .env up -d --build
docker compose ps
```
The catalog validates files during its background refresh and omits invalid
MP4s from the published snapshot. Use `./scripts/validate-media.sh "$MEDIA_DIR"`
before publishing only when an update must contain no invalid files at all.
For a media file at:
```text
/srv/media/movies/example.mp4
```
request this HLS master playlist:
```text
http://<vod-host>:8124/hls/movies/example.mp4/master.m3u8
```
Path segments must be URL encoded by clients. There is intentionally no
directory listing or progressive-download route.
## Media Catalog
Clients discover validated MP4 files through the same origin:
```text
GET /api/media
```
The response is a flat array from the latest completed catalog snapshot:
```json
[
{
"path": "movies/example.mp4",
"name": "example",
"playlistUrl": "/hls/movies/example.mp4/master.m3u8",
"coverUrl": "/api/media/covers/<cache-key>.jpg"
}
]
```
Catalog requests never scan media or run FFmpeg. A background worker scans the
tree at startup and then every six hours, validates one H.264 video track plus
one AAC audio track, and pre-generates covers for every valid file. It publishes
the completed JSON and JPEGs atomically, so clients continue to read the last
successful snapshot while a refresh runs or fails. Without any completed
snapshot, media and cover requests return `503`.
Use the `coverUrl` from the media item to request a pre-generated cover:
```text
GET /api/media/covers/<cache-key>.jpg
```
`GET /api/media/status` reports the refresh state, latest successful timestamp,
valid and skipped file counts, and the latest build error. Trigger an immediate
asynchronous rebuild with `POST /api/media/rescan`; concurrent requests do not
start duplicate builds. The service is intended for a trusted LAN, so this
endpoint has no additional authentication. Browser caching remains disabled.
```sh
curl -sS http://<vod-host>:8124/api/media/status
curl -i -X POST http://<vod-host>:8124/api/media/rescan
```
Run the smoke test after the container becomes healthy. Its default test media
is `data/aa/1170193193-1-192.mp4`; override it with a relative, URL-encoded
`TEST_MEDIA_PATH` when needed.
```sh
export PLAYER_ORIGIN=http://192.168.1.20:3000
./scripts/smoke-test.sh
```
## Publishing Updates
Use a staging directory on the same filesystem as the media library, validate
the staged MP4, and atomically rename it into place. Restart the container only
after the rename completes:
```sh
./scripts/validate-media.sh /srv/media/.staging
mv -T /srv/media/.staging/example.mp4 /srv/media/movies/example.mp4
./scripts/refresh-vod.sh
```
The service sends `Cache-Control: no-store` and the restart clears VOD caches.
This makes a replacement visible promptly, but intentionally interrupts active
playback for a short period. Do not overwrite MP4 files in place.
## Operations
- Health endpoint: `GET /healthz`
- Logs: `docker compose logs -f vod`
- Stop: `docker compose down`
- The host firewall should limit TCP `8124` to the trusted LAN. This service
does not provide TLS or authentication.
The first build downloads the fixed source inputs and base image. The ARM64
target host therefore needs access to Docker registries, `nginx.org`, and
GitHub during `docker compose build`.
+1
View File
@@ -0,0 +1 @@
"""Media catalog service package."""
+475
View File
@@ -0,0 +1,475 @@
#!/usr/bin/env python3
"""Prebuild and serve a read-only MP4 catalog snapshot."""
from __future__ import annotations
import hashlib
import json
import logging
import os
import re
import shutil
import subprocess
import threading
import uuid
from dataclasses import dataclass
from datetime import UTC, datetime
from http import HTTPStatus
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from urllib.parse import quote, urlsplit
LOG = logging.getLogger(__name__)
COVER_NAME = re.compile(r"^[0-9a-f]{64}\.jpg$")
DEFAULT_REFRESH_INTERVAL_SECONDS = 6 * 60 * 60
@dataclass(frozen=True)
class ProbeResult:
duration: float
attached_picture_stream: int | None
def inspect_media(source: Path) -> ProbeResult | None:
"""Validate a source file against the HLS service's supported streams."""
command = [
"ffprobe",
"-v",
"error",
"-show_entries",
"format=duration:stream=index,codec_name,codec_type:stream_disposition=attached_pic",
"-of",
"json",
str(source),
]
try:
result = subprocess.run(command, capture_output=True, check=False, text=True, timeout=30)
except (OSError, subprocess.TimeoutExpired) as error:
LOG.warning("Unable to inspect %s: %s", source, error)
return None
if result.returncode != 0:
LOG.warning("Ignoring unreadable media %s: %s", source, result.stderr.strip())
return None
try:
payload = json.loads(result.stdout)
duration = float(payload["format"]["duration"])
except (KeyError, TypeError, ValueError, json.JSONDecodeError):
LOG.warning("Ignoring media with incomplete metadata: %s", source)
return None
video_count = 0
audio_count = 0
attached_picture_stream: int | None = None
for stream in payload.get("streams", []):
codec_type = stream.get("codec_type")
codec_name = stream.get("codec_name")
attached_picture = stream.get("disposition", {}).get("attached_pic") == 1
if codec_type == "video" and attached_picture:
attached_picture_stream = stream.get("index")
elif codec_type == "video" and codec_name == "h264":
video_count += 1
elif codec_type == "audio" and codec_name == "aac":
audio_count += 1
else:
LOG.warning("Ignoring unsupported media %s", source)
return None
if video_count != 1 or audio_count != 1 or duration < 0:
LOG.warning("Ignoring unsupported media %s", source)
return None
return ProbeResult(duration=duration, attached_picture_stream=attached_picture_stream)
def playlist_url(relative_path: Path) -> str:
encoded_path = "/".join(quote(part, safe="") for part in relative_path.parts)
return f"/hls/{encoded_path}/master.m3u8"
def cache_key(relative_path: Path, stat_result: os.stat_result) -> str:
source_fingerprint = f"{relative_path.as_posix()}:{stat_result.st_size}:{stat_result.st_mtime_ns}"
return hashlib.sha256(source_fingerprint.encode("utf-8")).hexdigest()
class Catalog:
def __init__(
self,
media_dir: Path,
cache_dir: Path,
refresh_interval: float = DEFAULT_REFRESH_INTERVAL_SECONDS,
auto_refresh: bool = True,
) -> None:
self.media_dir = media_dir.resolve()
self.cache_dir = cache_dir.resolve()
self.releases_dir = self.cache_dir / "releases"
self.current_link = self.cache_dir / "current"
self.releases_dir.mkdir(parents=True, exist_ok=True)
self._state_lock = threading.Lock()
self._refresh_event = threading.Event()
self._stop_event = threading.Event()
self._build_active = False
self._refresh_pending = False
self._current_release: Path | None = None
self._media_payload: bytes | None = None
self._status: dict[str, object] = {
"state": "building",
"lastSuccessAt": None,
"validFiles": 0,
"skippedFiles": 0,
"lastError": None,
}
self._load_current_snapshot()
self._refresh_interval = refresh_interval
self._worker: threading.Thread | None = None
if auto_refresh:
self._worker = threading.Thread(target=self._worker_loop, name="catalog-refresh", daemon=True)
self._worker.start()
self.request_rescan()
def ready(self) -> bool:
with self._state_lock:
return self._media_payload is not None and self._current_release is not None
def media_payload(self) -> bytes | None:
with self._state_lock:
return self._media_payload
def cover_payload(self, cover_name: str) -> bytes | None:
if not COVER_NAME.fullmatch(cover_name):
return None
with self._state_lock:
release = self._current_release
if release is None:
return None
try:
return (release / "covers" / cover_name).read_bytes()
except OSError:
return None
def status(self) -> dict[str, object]:
with self._state_lock:
return {"ready": self._media_payload is not None, **self._status}
def request_rescan(self) -> dict[str, object]:
with self._state_lock:
if not self._build_active and not self._refresh_pending:
self._refresh_pending = True
self._status["state"] = "building"
self._status["lastError"] = None
self._refresh_event.set()
return {"ready": self._media_payload is not None, **self._status}
def refresh_now(self) -> None:
"""Synchronously build a snapshot. Intended for tests and worker use."""
with self._state_lock:
self._build_active = True
self._refresh_pending = False
self._status["state"] = "building"
self._status["lastError"] = None
try:
release, payload, metadata = self._build_snapshot()
self._publish_snapshot(release, payload, metadata)
except Exception as error: # Keep the previous release available on a failed build.
LOG.exception("Catalog refresh failed")
with self._state_lock:
self._status["state"] = "ready" if self._media_payload is not None else "failed"
self._status["lastError"] = str(error)
finally:
with self._state_lock:
self._build_active = False
def _worker_loop(self) -> None:
while not self._stop_event.is_set():
requested = self._refresh_event.wait(self._refresh_interval)
if self._stop_event.is_set():
return
self._refresh_event.clear()
with self._state_lock:
if not requested and not self._refresh_pending:
self._refresh_pending = True
self._status["state"] = "building"
if self._build_active or not self._refresh_pending:
continue
self.refresh_now()
def _build_snapshot(self) -> tuple[Path, bytes, dict[str, object]]:
release_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex}"
staging = self.releases_dir / f".staging-{release_id}"
release = self.releases_dir / release_id
covers_dir = staging / "covers"
covers_dir.mkdir(parents=True)
valid_files = 0
skipped_files = 0
items: list[dict[str, str]] = []
try:
for source in self._media_files():
try:
relative_path = source.relative_to(self.media_dir)
before = source.stat()
except OSError:
skipped_files += 1
continue
probe = inspect_media(source)
if probe is None:
skipped_files += 1
continue
cover_name = f"{cache_key(relative_path, before)}.jpg"
if not self._generate_cover(source, covers_dir / cover_name, probe):
skipped_files += 1
continue
try:
after = source.stat()
except OSError:
skipped_files += 1
continue
if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
LOG.warning("Ignoring media changed while processing: %s", source)
(covers_dir / cover_name).unlink(missing_ok=True)
skipped_files += 1
continue
valid_files += 1
items.append(
{
"path": relative_path.as_posix(),
"name": source.stem,
"playlistUrl": playlist_url(relative_path),
"coverUrl": f"/api/media/covers/{cover_name}",
}
)
items.sort(key=lambda item: (item["name"].casefold(), item["path"].casefold()))
payload = json.dumps(items, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
generated_at = datetime.now(UTC).isoformat().replace("+00:00", "Z")
metadata: dict[str, object] = {
"generatedAt": generated_at,
"validFiles": valid_files,
"skippedFiles": skipped_files,
}
(staging / "media.json").write_bytes(payload)
(staging / "metadata.json").write_text(json.dumps(metadata, separators=(",", ":")), encoding="utf-8")
staging.replace(release)
return release, payload, metadata
except Exception:
shutil.rmtree(staging, ignore_errors=True)
raise
def _publish_snapshot(self, release: Path, payload: bytes, metadata: dict[str, object]) -> None:
pending_link = self.cache_dir / f".current-{uuid.uuid4().hex}"
os.symlink(str(release.relative_to(self.cache_dir)), pending_link)
try:
os.replace(pending_link, self.current_link)
finally:
pending_link.unlink(missing_ok=True)
with self._state_lock:
self._current_release = release
self._media_payload = payload
self._status.update(
{
"state": "ready",
"lastSuccessAt": metadata["generatedAt"],
"validFiles": metadata["validFiles"],
"skippedFiles": metadata["skippedFiles"],
"lastError": None,
}
)
self._cleanup_old_releases(release)
def _load_current_snapshot(self) -> None:
try:
release = self.current_link.resolve(strict=True)
payload = (release / "media.json").read_bytes()
items = json.loads(payload)
metadata = json.loads((release / "metadata.json").read_text(encoding="utf-8"))
if not isinstance(items, list) or not isinstance(metadata, dict):
raise ValueError("invalid snapshot")
except (OSError, ValueError, json.JSONDecodeError) as error:
if self.current_link.exists() or self.current_link.is_symlink():
LOG.warning("Ignoring invalid catalog snapshot: %s", error)
return
self._current_release = release
self._media_payload = payload
self._status.update(
{
"state": "ready",
"lastSuccessAt": metadata.get("generatedAt"),
"validFiles": metadata.get("validFiles", 0),
"skippedFiles": metadata.get("skippedFiles", 0),
}
)
def _cleanup_old_releases(self, active_release: Path) -> None:
releases = sorted(
(path for path in self.releases_dir.iterdir() if path.is_dir() and not path.name.startswith(".staging-")),
key=lambda path: path.stat().st_mtime_ns,
reverse=True,
)
keep = {active_release, *releases[:2]}
for release in releases:
if release not in keep:
shutil.rmtree(release, ignore_errors=True)
def _generate_cover(self, source: Path, cover_path: Path, probe: ProbeResult) -> bool:
temporary_path = cover_path.with_name(f".{cover_path.stem}.{os.getpid()}.jpg")
if probe.attached_picture_stream is not None:
command = [
"ffmpeg",
"-nostdin",
"-v",
"error",
"-i",
str(source),
"-map",
f"0:{probe.attached_picture_stream}",
"-frames:v",
"1",
"-vf",
"scale=480:-2",
"-q:v",
"2",
"-y",
str(temporary_path),
]
else:
timestamp = min(10.0, probe.duration * 0.1)
command = [
"ffmpeg",
"-nostdin",
"-v",
"error",
"-ss",
f"{timestamp:.3f}",
"-i",
str(source),
"-frames:v",
"1",
"-vf",
"scale=480:-2",
"-q:v",
"2",
"-y",
str(temporary_path),
]
try:
result = subprocess.run(command, capture_output=True, check=False, text=True, timeout=60)
if result.returncode != 0 or not temporary_path.is_file() or temporary_path.stat().st_size == 0:
LOG.warning("Unable to generate cover for %s: %s", source, result.stderr.strip())
return False
temporary_path.replace(cover_path)
return True
except (OSError, subprocess.TimeoutExpired) as error:
LOG.warning("Unable to generate cover for %s: %s", source, error)
return False
finally:
temporary_path.unlink(missing_ok=True)
def _media_files(self) -> list[Path]:
files: list[Path] = []
for root, directories, filenames in os.walk(self.media_dir, followlinks=False):
directories[:] = [name for name in directories if not (Path(root) / name).is_symlink()]
for filename in filenames:
source = Path(root) / filename
if source.suffix.lower() == ".mp4" and source.is_file() and not source.is_symlink():
files.append(source)
return sorted(files, key=lambda path: path.relative_to(self.media_dir).as_posix().casefold())
class CatalogRequestHandler(BaseHTTPRequestHandler):
server: "CatalogHTTPServer"
def do_GET(self) -> None: # noqa: N802
request = urlsplit(self.path)
if request.path == "/healthz":
if self.server.catalog.ready():
self._send_bytes(HTTPStatus.OK, b"ok\n", "text/plain; charset=utf-8")
else:
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"error": "catalog is building"})
elif request.path == "/api/media/status":
self._send_json(HTTPStatus.OK, self.server.catalog.status())
elif request.path == "/api/media":
payload = self.server.catalog.media_payload()
if payload is None:
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"error": "catalog is building"})
else:
self._send_bytes(HTTPStatus.OK, payload, "application/json; charset=utf-8")
elif request.path.startswith("/api/media/covers/"):
if not self.server.catalog.ready():
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"error": "catalog is building"})
else:
self._send_cover(request.path.rsplit("/", 1)[-1])
else:
self.send_error(HTTPStatus.NOT_FOUND)
def do_HEAD(self) -> None: # noqa: N802
self.do_GET()
def do_POST(self) -> None: # noqa: N802
if urlsplit(self.path).path != "/api/media/rescan":
self.send_error(HTTPStatus.NOT_FOUND)
return
self._send_json(HTTPStatus.ACCEPTED, self.server.catalog.request_rescan())
def do_OPTIONS(self) -> None: # noqa: N802
self.send_response(HTTPStatus.NO_CONTENT)
self.send_header("Allow", "GET, HEAD, POST, OPTIONS")
self.end_headers()
def _send_cover(self, cover_name: str) -> None:
payload = self.server.catalog.cover_payload(cover_name)
if payload is None:
self.send_error(HTTPStatus.NOT_FOUND)
return
self._send_bytes(HTTPStatus.OK, payload, "image/jpeg")
def _send_json(self, status: HTTPStatus, payload: object) -> None:
self._send_bytes(
status,
json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
"application/json; charset=utf-8",
)
def _send_bytes(self, status: HTTPStatus, body: bytes, content_type: str) -> None:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
self.send_header("Cache-Control", "no-store")
self.end_headers()
if self.command != "HEAD":
self.wfile.write(body)
def log_message(self, format_string: str, *args: object) -> None:
LOG.info("%s - %s", self.address_string(), format_string % args)
class CatalogHTTPServer(ThreadingHTTPServer):
daemon_threads = True
def __init__(self, address: tuple[str, int], catalog: Catalog) -> None:
super().__init__(address, CatalogRequestHandler)
self.catalog = catalog
def main() -> None:
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s")
media_dir = Path(os.environ.get("MEDIA_DIR", "/media"))
cache_dir = Path(os.environ.get("CATALOG_CACHE_DIR", "/cache"))
refresh_interval = float(os.environ.get("CATALOG_REFRESH_INTERVAL_SECONDS", str(DEFAULT_REFRESH_INTERVAL_SECONDS)))
port = int(os.environ.get("CATALOG_PORT", "8081"))
if not media_dir.is_dir():
raise SystemExit(f"Media directory does not exist: {media_dir}")
if refresh_interval <= 0:
raise SystemExit("CATALOG_REFRESH_INTERVAL_SECONDS must be positive")
CatalogHTTPServer(("0.0.0.0", port), Catalog(media_dir, cache_dir, refresh_interval)).serve_forever()
if __name__ == "__main__":
main()
+69
View File
@@ -0,0 +1,69 @@
services:
vod:
build:
context: .
target: vod
args:
NGINX_VERSION: "${NGINX_VERSION:-1.30.5}"
KALTURA_VOD_REF: "${KALTURA_VOD_REF:-26f06877b0f2a2336e59cda93a3de18d7b23a3e2}"
image: "${IMAGE_NAME:-nginx-hls-vod}:local"
environment:
PLAYER_ORIGIN: "${PLAYER_ORIGIN:?Set PLAYER_ORIGIN in .env}"
depends_on:
catalog:
condition: service_healthy
ports:
- "${HLS_BIND_ADDRESS:-0.0.0.0}:${HLS_PORT:-8124}:8080"
volumes:
- type: bind
source: "${MEDIA_DIR:?Set MEDIA_DIR in .env}"
target: /media
read_only: true
bind:
create_host_path: false
restart: unless-stopped
read_only: true
tmpfs:
- /tmp:rw,nosuid,nodev,noexec,size=64m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
mem_limit: 512m
cpus: 2.0
catalog:
build:
context: .
target: catalog
image: "${CATALOG_IMAGE:-nginx-hls-catalog}:local"
user: "0:0"
environment:
MEDIA_DIR: /media
CATALOG_CACHE_DIR: /cache
CATALOG_REFRESH_INTERVAL_SECONDS: "${CATALOG_REFRESH_INTERVAL_SECONDS:-21600}"
CATALOG_PORT: "8081"
volumes:
- type: bind
source: "${MEDIA_DIR:?Set MEDIA_DIR in .env}"
target: /media
read_only: true
bind:
create_host_path: false
- type: bind
source: "${CATALOG_CACHE_DIR:?Set CATALOG_CACHE_DIR in .env}"
target: /cache
bind:
create_host_path: false
restart: unless-stopped
read_only: true
tmpfs:
- /tmp:rw,nosuid,nodev,noexec,size=64m
cap_drop:
- ALL
security_opt:
- no-new-privileges:true
pids_limit: 128
mem_limit: 512m
cpus: 2.0
+195
View File
@@ -0,0 +1,195 @@
worker_processes auto;
error_log /dev/stderr warn;
pid /tmp/nginx.pid;
events {
worker_connections 1024;
}
thread_pool vod_io threads=2 max_queue=65536;
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format vod '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" rt=$request_time';
access_log /dev/stdout vod;
sendfile on;
tcp_nopush on;
keepalive_timeout 65;
server_tokens off;
aio threads=vod_io;
upstream catalog_backend {
server catalog:8081;
}
server {
listen 8080;
server_name _;
autoindex off;
vod_mode local;
vod_metadata_cache metadata_cache 64m;
vod_response_cache response_cache 16m;
vod_open_file_thread_pool vod_io;
open_file_cache max=1000 inactive=30s;
open_file_cache_valid 30s;
open_file_cache_min_uses 1;
open_file_cache_errors on;
location = /healthz {
access_log off;
default_type text/plain;
return 200 "ok\n";
}
location ^~ /hls/ {
alias /media/;
vod hls;
vod_segment_duration 6000;
vod_align_segments_to_key_frames on;
vod_manifest_segment_durations_mode accurate;
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
add_header Access-Control-Expose-Headers "Accept-Ranges, Content-Length, Content-Range, Content-Type" always;
add_header Cache-Control "no-store" always;
add_header Vary "Origin" always;
if ($request_method = OPTIONS) {
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
add_header Access-Control-Max-Age 600 always;
add_header Cache-Control "no-store" always;
add_header Vary "Origin" always;
return 204;
}
if ($request_method !~ ^(GET|HEAD)$) {
return 405;
}
}
location = /api/media {
proxy_pass http://catalog_backend;
proxy_set_header Host $host;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_hide_header Cache-Control;
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
add_header Cache-Control "no-store" always;
add_header Vary "Origin" always;
if ($request_method = OPTIONS) {
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
add_header Access-Control-Max-Age 600 always;
add_header Cache-Control "no-store" always;
add_header Vary "Origin" always;
return 204;
}
if ($request_method !~ ^(GET|HEAD)$) {
return 405;
}
}
location = /api/media/status {
proxy_pass http://catalog_backend;
proxy_set_header Host $host;
proxy_hide_header Cache-Control;
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
add_header Cache-Control "no-store" always;
add_header Vary "Origin" always;
if ($request_method = OPTIONS) {
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
add_header Access-Control-Max-Age 600 always;
add_header Cache-Control "no-store" always;
add_header Vary "Origin" always;
return 204;
}
if ($request_method !~ ^(GET|HEAD)$) {
return 405;
}
}
location = /api/media/rescan {
proxy_pass http://catalog_backend;
proxy_set_header Host $host;
proxy_hide_header Cache-Control;
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
add_header Access-Control-Allow-Methods "POST, OPTIONS" always;
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
add_header Cache-Control "no-store" always;
add_header Vary "Origin" always;
if ($request_method = OPTIONS) {
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
add_header Access-Control-Allow-Methods "POST, OPTIONS" always;
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
add_header Access-Control-Max-Age 600 always;
add_header Cache-Control "no-store" always;
add_header Vary "Origin" always;
return 204;
}
if ($request_method != POST) {
return 405;
}
}
location ^~ /api/media/covers/ {
proxy_pass http://catalog_backend;
proxy_set_header Host $host;
proxy_hide_header Cache-Control;
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
add_header Cache-Control "no-store" always;
add_header Vary "Origin" always;
if ($request_method = OPTIONS) {
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
add_header Access-Control-Max-Age 600 always;
add_header Cache-Control "no-store" always;
add_header Vary "Origin" always;
return 204;
}
if ($request_method !~ ^(GET|HEAD)$) {
return 405;
}
}
location / {
return 404;
}
}
}
+17
View File
@@ -0,0 +1,17 @@
#!/bin/sh
set -eu
: "${PLAYER_ORIGIN:?PLAYER_ORIGIN must be set}"
if ! printf '%s' "$PLAYER_ORIGIN" | grep -Eq '^https?://[A-Za-z0-9._:-]+$'; then
echo "PLAYER_ORIGIN must be an origin such as http://192.168.1.20:3000" >&2
exit 64
fi
envsubst '${PLAYER_ORIGIN}' \
< /etc/nginx/templates/nginx.conf.template \
> /tmp/nginx.conf
nginx -t -c /tmp/nginx.conf
exec nginx -c /tmp/nginx.conf -g 'daemon off;'
+6
View File
@@ -0,0 +1,6 @@
#!/usr/bin/env bash
set -euo pipefail
echo "Restarting VOD after an already-completed atomic media update."
docker compose restart vod
+52
View File
@@ -0,0 +1,52 @@
#!/usr/bin/env bash
set -euo pipefail
: "${PLAYER_ORIGIN:?Set PLAYER_ORIGIN before running this test}"
base_url=${HLS_BASE_URL:-http://127.0.0.1:8124}
media_path=${TEST_MEDIA_PATH:-aa/1170193193-1-192.mp4}
tmpdir=$(mktemp -d)
trap 'rm -rf "$tmpdir"' EXIT
resolve_url() {
local base=$1
local reference=$2
case "$reference" in
http://*|https://*) printf '%s\n' "$reference" ;;
/*) printf '%s%s\n' "$(printf '%s\n' "$base" | sed -E 's#(https?://[^/]+).*#\1#')" "$reference" ;;
*) printf '%s/%s\n' "${base%/*}" "$reference" ;;
esac
}
master_url="${base_url%/}/hls/${media_path}/master.m3u8"
master_headers="$tmpdir/master.headers"
master_playlist="$tmpdir/master.m3u8"
curl --fail --silent --show-error --dump-header "$master_headers" --output "$master_playlist" "$master_url"
tr -d '\r' < "$master_headers" | grep -Fxi "Access-Control-Allow-Origin: $PLAYER_ORIGIN" >/dev/null
tr -d '\r' < "$master_headers" | grep -Fxi 'Cache-Control: no-store' >/dev/null
grep -Fqx '#EXTM3U' "$master_playlist" >/dev/null
child_reference=$(awk 'found { print; exit } /^#EXT-X-STREAM-INF/ { found=1 }' "$master_playlist")
if [[ -z $child_reference ]]; then
child_url=$master_url
else
child_url=$(resolve_url "$master_url" "$child_reference")
fi
child_playlist="$tmpdir/index.m3u8"
curl --fail --silent --show-error --output "$child_playlist" "$child_url"
grep -Fqx '#EXTM3U' "$child_playlist" >/dev/null
segment_reference=$(awk '!/^#/ && NF { print; exit }' "$child_playlist")
if [[ -z $segment_reference ]]; then
echo "No HLS segment was found in $child_url" >&2
exit 1
fi
segment_url=$(resolve_url "$child_url" "$segment_reference")
curl --fail --silent --show-error --output /dev/null "$segment_url"
printf 'HLS smoke test passed: %s\n' "$master_url"
+100
View File
@@ -0,0 +1,100 @@
#!/usr/bin/env bash
set -euo pipefail
media_root=${1:-data}
if ! command -v ffprobe >/dev/null 2>&1; then
echo "ffprobe is required to validate media." >&2
exit 2
fi
if [[ ! -d $media_root ]]; then
printf 'Media directory does not exist: %s\n' "$media_root" >&2
exit 2
fi
valid_count=0
invalid_count=0
validate_file() {
local file=$1
local format_name streams codec type attached_pic
local video_count=0
local audio_count=0
local invalid_reason=
if ! format_name=$(ffprobe -v error -show_entries format=format_name \
-of default=noprint_wrappers=1:nokey=1 "$file"); then
printf 'INVALID: %q: unreadable or corrupt MP4\n' "$file" >&2
return 1
fi
if [[ ,$format_name, != *,mp4,* ]]; then
printf 'INVALID: %q: expected an MP4 container, got %s\n' "$file" "$format_name" >&2
return 1
fi
if ! streams=$(ffprobe -v error \
-show_entries stream=codec_name,codec_type:stream_disposition=attached_pic \
-of csv=p=0 "$file"); then
printf 'INVALID: %q: unable to inspect streams\n' "$file" >&2
return 1
fi
while IFS=, read -r codec type attached_pic; do
case "$type" in
video)
if [[ $attached_pic == 1 ]]; then
continue
fi
if [[ $codec != h264 ]]; then
invalid_reason="unsupported video codec $codec"
fi
((video_count += 1))
;;
audio)
if [[ $codec != aac ]]; then
invalid_reason="unsupported audio codec $codec"
fi
((audio_count += 1))
;;
*)
invalid_reason="unsupported $type track ($codec)"
;;
esac
done <<< "$streams"
if [[ -n $invalid_reason ]]; then
printf 'INVALID: %q: %s\n' "$file" "$invalid_reason" >&2
return 1
fi
if ((video_count != 1 || audio_count != 1)); then
printf 'INVALID: %q: expected one H.264 video and one AAC audio track; found %d video and %d audio\n' \
"$file" "$video_count" "$audio_count" >&2
return 1
fi
printf 'VALID: %s\n' "$file"
}
while IFS= read -r -d '' file; do
if validate_file "$file"; then
((valid_count += 1))
else
((invalid_count += 1))
fi
done < <(find "$media_root" -type f -iname '*.mp4' -print0)
if ((valid_count == 0)); then
echo "No valid MP4 files were found." >&2
exit 1
fi
if ((invalid_count > 0)); then
printf 'Validation failed: %d valid, %d invalid file(s).\n' "$valid_count" "$invalid_count" >&2
exit 1
fi
printf 'Validation passed: %d MP4 file(s).\n' "$valid_count"
+153
View File
@@ -0,0 +1,153 @@
import json
import subprocess
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from catalog.app import Catalog, ProbeResult, cache_key, inspect_media, playlist_url
class CatalogTests(unittest.TestCase):
def test_playlist_url_encodes_each_path_segment(self) -> None:
self.assertEqual(
playlist_url(Path("中文 media") / "clip #1.mp4"),
"/hls/%E4%B8%AD%E6%96%87%20media/clip%20%231.mp4/master.m3u8",
)
def test_cache_key_changes_when_media_changes(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
source = Path(temporary_directory) / "clip.mp4"
source.write_bytes(b"a")
first = cache_key(Path("clip.mp4"), source.stat())
source.write_bytes(b"updated")
second = cache_key(Path("clip.mp4"), source.stat())
self.assertNotEqual(first, second)
def test_refresh_publishes_validated_list_and_cached_cover(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
root = Path(temporary_directory)
media_dir = root / "media"
cache_dir = root / "cache"
media_dir.mkdir()
source = media_dir / "clip.mp4"
source.write_bytes(b"video")
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
probe_payload = json.dumps(
{
"format": {"duration": "20"},
"streams": [
{"index": 0, "codec_type": "video", "codec_name": "h264", "disposition": {"attached_pic": 0}},
{"index": 1, "codec_type": "audio", "codec_name": "aac", "disposition": {"attached_pic": 0}},
],
}
)
def fake_run(command: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]:
if command[0] == "ffprobe":
return subprocess.CompletedProcess(command, 0, probe_payload, "")
Path(command[-1]).write_bytes(b"jpeg")
return subprocess.CompletedProcess(command, 0, "", "")
self.assertFalse(catalog.ready())
with patch("catalog.app.subprocess.run", side_effect=fake_run) as run:
catalog.refresh_now()
self.assertTrue(catalog.ready())
self.assertTrue((cache_dir / "current").is_symlink())
items = json.loads(catalog.media_payload())
self.assertEqual(len(items), 1)
self.assertEqual(items[0]["path"], "clip.mp4")
self.assertEqual(items[0]["playlistUrl"], "/hls/clip.mp4/master.m3u8")
cover_name = items[0]["coverUrl"].rsplit("/", 1)[-1]
self.assertEqual(catalog.cover_payload(cover_name), b"jpeg")
self.assertEqual(run.call_count, 2)
with patch("catalog.app.subprocess.run") as run:
self.assertEqual(json.loads(catalog.media_payload()), items)
self.assertEqual(catalog.cover_payload(cover_name), b"jpeg")
run.assert_not_called()
def test_refresh_excludes_invalid_media(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
root = Path(temporary_directory)
media_dir = root / "media"
cache_dir = root / "cache"
media_dir.mkdir()
(media_dir / "valid.mp4").write_bytes(b"valid")
(media_dir / "broken.mp4").write_bytes(b"broken")
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
def fake_probe(source: Path) -> ProbeResult | None:
return ProbeResult(10.0, None) if source.name == "valid.mp4" else None
with patch("catalog.app.inspect_media", side_effect=fake_probe), patch.object(
Catalog, "_generate_cover", return_value=True
):
catalog.refresh_now()
items = json.loads(catalog.media_payload())
self.assertEqual([item["path"] for item in items], ["valid.mp4"])
status = catalog.status()
self.assertEqual(status["validFiles"], 1)
self.assertEqual(status["skippedFiles"], 1)
def test_failed_refresh_keeps_previous_snapshot(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
root = Path(temporary_directory)
media_dir = root / "media"
cache_dir = root / "cache"
media_dir.mkdir()
(media_dir / "clip.mp4").write_bytes(b"video")
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
with patch("catalog.app.inspect_media", return_value=ProbeResult(10.0, None)), patch.object(
Catalog, "_generate_cover", return_value=True
):
catalog.refresh_now()
previous_payload = catalog.media_payload()
with patch.object(Catalog, "_build_snapshot", side_effect=RuntimeError("disk full")):
catalog.refresh_now()
self.assertTrue(catalog.ready())
self.assertEqual(catalog.media_payload(), previous_payload)
self.assertEqual(catalog.status()["state"], "ready")
self.assertEqual(catalog.status()["lastError"], "disk full")
def test_existing_snapshot_is_loaded_without_subprocesses(self) -> None:
with tempfile.TemporaryDirectory() as temporary_directory:
root = Path(temporary_directory)
media_dir = root / "media"
cache_dir = root / "cache"
media_dir.mkdir()
(media_dir / "clip.mp4").write_bytes(b"video")
builder = Catalog(media_dir, cache_dir, auto_refresh=False)
with patch("catalog.app.inspect_media", return_value=ProbeResult(10.0, None)), patch.object(
Catalog, "_generate_cover", return_value=True
):
builder.refresh_now()
with patch("catalog.app.subprocess.run") as run:
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
payload = catalog.media_payload()
run.assert_not_called()
self.assertTrue(catalog.ready())
self.assertEqual(len(json.loads(payload)), 1)
def test_inspect_media_rejects_unsupported_streams(self) -> None:
payload = {
"format": {"duration": "12.5"},
"streams": [
{"index": 0, "codec_type": "video", "codec_name": "h264", "disposition": {"attached_pic": 0}},
{"index": 1, "codec_type": "audio", "codec_name": "aac", "disposition": {"attached_pic": 0}},
{"index": 2, "codec_type": "subtitle", "codec_name": "mov_text", "disposition": {"attached_pic": 0}},
],
}
completed = subprocess.CompletedProcess([], 0, json.dumps(payload), "")
with patch("catalog.app.subprocess.run", return_value=completed):
self.assertIsNone(inspect_media(Path("clip.mp4")))
if __name__ == "__main__":
unittest.main()