init
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
.env
|
||||
.git
|
||||
.gitignore
|
||||
data
|
||||
catalog-cache
|
||||
README.md
|
||||
scripts
|
||||
@@ -0,0 +1,19 @@
|
||||
# Exact browser origin allowed to request HLS resources. This value is required.
|
||||
PLAYER_ORIGIN=http://192.168.1.20:3000
|
||||
|
||||
# Development uses the repository test library. Production must use an absolute host path.
|
||||
MEDIA_DIR=./data
|
||||
|
||||
# Persistent host directory for catalog JSON snapshots and generated JPEG covers.
|
||||
# Create it before starting the stack.
|
||||
CATALOG_CACHE_DIR=./catalog-cache
|
||||
CATALOG_REFRESH_INTERVAL_SECONDS=21600
|
||||
|
||||
# LAN HTTP listener published by Docker Compose.
|
||||
HLS_BIND_ADDRESS=0.0.0.0
|
||||
HLS_PORT=8124
|
||||
|
||||
# Fixed build inputs. Upgrade these deliberately and re-run the smoke test.
|
||||
NGINX_VERSION=1.30.5
|
||||
KALTURA_VOD_REF=26f06877b0f2a2336e59cda93a3de18d7b23a3e2
|
||||
IMAGE_NAME=nginx-hls-vod
|
||||
+30
@@ -0,0 +1,30 @@
|
||||
# Local configuration and runtime catalog cache
|
||||
.env
|
||||
.env.*
|
||||
!.env.example
|
||||
catalog-cache/
|
||||
|
||||
# Python caches, virtual environments, and test coverage
|
||||
__pycache__/
|
||||
*.py[cod]
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
coverage.xml
|
||||
htmlcov/
|
||||
.venv/
|
||||
venv/
|
||||
env/
|
||||
|
||||
# Local Docker overrides and editor/OS files
|
||||
compose.override.yml
|
||||
compose.override.yaml
|
||||
docker-compose.override.yml
|
||||
*.log
|
||||
.DS_Store
|
||||
Thumbs.db
|
||||
.idea/
|
||||
*.iml
|
||||
|
||||
# Test media stays local and is never sent as a Docker build context.
|
||||
data/*
|
||||
!data/.gitkeep
|
||||
+96
@@ -0,0 +1,96 @@
|
||||
FROM debian:bookworm-slim AS builder
|
||||
|
||||
ARG NGINX_VERSION=1.30.5
|
||||
ARG KALTURA_VOD_REF=26f06877b0f2a2336e59cda93a3de18d7b23a3e2
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
ca-certificates \
|
||||
curl \
|
||||
libpcre2-dev \
|
||||
zlib1g-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
RUN mkdir -p /src/nginx-vod-module \
|
||||
&& curl -fsSL "https://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz" -o /tmp/nginx.tar.gz \
|
||||
&& curl -fsSL "https://codeload.github.com/kaltura/nginx-vod-module/tar.gz/${KALTURA_VOD_REF}" -o /tmp/nginx-vod-module.tar.gz \
|
||||
&& tar -xzf /tmp/nginx.tar.gz -C /src \
|
||||
&& tar -xzf /tmp/nginx-vod-module.tar.gz -C /src/nginx-vod-module --strip-components=1 \
|
||||
&& rm -f /tmp/nginx.tar.gz /tmp/nginx-vod-module.tar.gz
|
||||
|
||||
WORKDIR /src/nginx-${NGINX_VERSION}
|
||||
|
||||
RUN ./configure \
|
||||
--prefix=/usr/share/nginx \
|
||||
--sbin-path=/usr/sbin/nginx \
|
||||
--conf-path=/etc/nginx/nginx.conf \
|
||||
--pid-path=/tmp/nginx.pid \
|
||||
--error-log-path=/dev/stderr \
|
||||
--http-log-path=/dev/stdout \
|
||||
--http-client-body-temp-path=/tmp/client_body \
|
||||
--http-proxy-temp-path=/tmp/proxy \
|
||||
--http-fastcgi-temp-path=/tmp/fastcgi \
|
||||
--http-uwsgi-temp-path=/tmp/uwsgi \
|
||||
--http-scgi-temp-path=/tmp/scgi \
|
||||
--with-file-aio \
|
||||
--with-threads \
|
||||
--with-http_gzip_static_module \
|
||||
--with-http_stub_status_module \
|
||||
--with-cc-opt='-O2' \
|
||||
--add-module=/src/nginx-vod-module \
|
||||
&& make -j"$(nproc)" \
|
||||
&& make install
|
||||
|
||||
FROM debian:bookworm-slim AS vod
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ca-certificates \
|
||||
curl \
|
||||
gettext-base \
|
||||
libpcre2-8-0 \
|
||||
zlib1g \
|
||||
&& addgroup --system nginx \
|
||||
&& adduser --system --ingroup nginx --no-create-home --home /nonexistent nginx \
|
||||
&& mkdir -p /etc/nginx/templates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=builder /usr/sbin/nginx /usr/sbin/nginx
|
||||
COPY --from=builder /etc/nginx/mime.types /etc/nginx/mime.types
|
||||
COPY config/nginx.conf.template /etc/nginx/templates/nginx.conf.template
|
||||
COPY docker/entrypoint.sh /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
RUN chmod 0755 /usr/local/bin/docker-entrypoint.sh
|
||||
|
||||
USER nginx
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \
|
||||
CMD curl --fail --silent http://127.0.0.1:8080/healthz || exit 1
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/docker-entrypoint.sh"]
|
||||
|
||||
FROM debian:bookworm-slim AS catalog
|
||||
|
||||
RUN apt-get update \
|
||||
&& apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
python3 \
|
||||
&& addgroup --system catalog \
|
||||
&& adduser --system --ingroup catalog --no-create-home --home /nonexistent catalog \
|
||||
&& mkdir -p /app /cache \
|
||||
&& chown catalog:catalog /app /cache \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY catalog/app.py /app/app.py
|
||||
|
||||
USER catalog
|
||||
|
||||
EXPOSE 8081
|
||||
|
||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \
|
||||
CMD python3 -c "from urllib.request import urlopen; urlopen('http://127.0.0.1:8081/healthz', timeout=3).read()" || exit 1
|
||||
|
||||
ENTRYPOINT ["python3", "/app/app.py"]
|
||||
@@ -0,0 +1,159 @@
|
||||
# NGINX HLS VOD
|
||||
|
||||
Docker Compose deployment for local MP4 to HLS VOD packaging using Kaltura
|
||||
`nginx-vod-module`. The service maps a media tree read-only into the container
|
||||
and preserves its hierarchy in HLS URLs.
|
||||
|
||||
## Scope
|
||||
|
||||
- ARM64 Linux Docker host
|
||||
- Trusted LAN HTTP service on port `8124`
|
||||
- One H.264 video track and one AAC audio track per MP4
|
||||
- One configured browser origin for CORS
|
||||
- No authentication, TLS, ABR, external subtitles, or multiple audio/video
|
||||
tracks
|
||||
|
||||
An attached-picture video stream such as MP4 cover art is ignored by the media
|
||||
validator.
|
||||
|
||||
## Configuration
|
||||
|
||||
Copy the environment template and replace the example player origin:
|
||||
|
||||
```sh
|
||||
cp .env.example .env
|
||||
```
|
||||
|
||||
Required values:
|
||||
|
||||
```dotenv
|
||||
PLAYER_ORIGIN=http://192.168.1.20:3000
|
||||
MEDIA_DIR=/srv/media
|
||||
CATALOG_CACHE_DIR=/srv/nginx-hls/catalog-cache
|
||||
HLS_PORT=8124
|
||||
```
|
||||
|
||||
`MEDIA_DIR=./data` is appropriate only for local development. Production uses
|
||||
an absolute path on the Docker host. The container user must be able to read
|
||||
the mounted files and directories. `CATALOG_CACHE_DIR` stores catalog snapshots
|
||||
and generated covers. Create it before startup, for example
|
||||
`install -d -m 0700 /srv/nginx-hls/catalog-cache`.
|
||||
`CATALOG_REFRESH_INTERVAL_SECONDS` defaults to `21600` (6 hours).
|
||||
|
||||
The Dockerfile deliberately fixes NGINX `1.30.5` and Kaltura module commit
|
||||
`26f06877b0f2a2336e59cda93a3de18d7b23a3e2`. Change either only as a deliberate
|
||||
upgrade, then rebuild and run the smoke test.
|
||||
|
||||
## Run
|
||||
|
||||
Start the stack after creating the configured cache directory:
|
||||
|
||||
```sh
|
||||
set -a
|
||||
. ./.env
|
||||
set +a
|
||||
install -d -m 0700 "$CATALOG_CACHE_DIR"
|
||||
docker compose --env-file .env up -d --build
|
||||
docker compose ps
|
||||
```
|
||||
|
||||
The catalog validates files during its background refresh and omits invalid
|
||||
MP4s from the published snapshot. Use `./scripts/validate-media.sh "$MEDIA_DIR"`
|
||||
before publishing only when an update must contain no invalid files at all.
|
||||
|
||||
For a media file at:
|
||||
|
||||
```text
|
||||
/srv/media/movies/example.mp4
|
||||
```
|
||||
|
||||
request this HLS master playlist:
|
||||
|
||||
```text
|
||||
http://<vod-host>:8124/hls/movies/example.mp4/master.m3u8
|
||||
```
|
||||
|
||||
Path segments must be URL encoded by clients. There is intentionally no
|
||||
directory listing or progressive-download route.
|
||||
|
||||
## Media Catalog
|
||||
|
||||
Clients discover validated MP4 files through the same origin:
|
||||
|
||||
```text
|
||||
GET /api/media
|
||||
```
|
||||
|
||||
The response is a flat array from the latest completed catalog snapshot:
|
||||
|
||||
```json
|
||||
[
|
||||
{
|
||||
"path": "movies/example.mp4",
|
||||
"name": "example",
|
||||
"playlistUrl": "/hls/movies/example.mp4/master.m3u8",
|
||||
"coverUrl": "/api/media/covers/<cache-key>.jpg"
|
||||
}
|
||||
]
|
||||
```
|
||||
|
||||
Catalog requests never scan media or run FFmpeg. A background worker scans the
|
||||
tree at startup and then every six hours, validates one H.264 video track plus
|
||||
one AAC audio track, and pre-generates covers for every valid file. It publishes
|
||||
the completed JSON and JPEGs atomically, so clients continue to read the last
|
||||
successful snapshot while a refresh runs or fails. Without any completed
|
||||
snapshot, media and cover requests return `503`.
|
||||
|
||||
Use the `coverUrl` from the media item to request a pre-generated cover:
|
||||
|
||||
```text
|
||||
GET /api/media/covers/<cache-key>.jpg
|
||||
```
|
||||
|
||||
`GET /api/media/status` reports the refresh state, latest successful timestamp,
|
||||
valid and skipped file counts, and the latest build error. Trigger an immediate
|
||||
asynchronous rebuild with `POST /api/media/rescan`; concurrent requests do not
|
||||
start duplicate builds. The service is intended for a trusted LAN, so this
|
||||
endpoint has no additional authentication. Browser caching remains disabled.
|
||||
|
||||
```sh
|
||||
curl -sS http://<vod-host>:8124/api/media/status
|
||||
curl -i -X POST http://<vod-host>:8124/api/media/rescan
|
||||
```
|
||||
|
||||
Run the smoke test after the container becomes healthy. Its default test media
|
||||
is `data/aa/1170193193-1-192.mp4`; override it with a relative, URL-encoded
|
||||
`TEST_MEDIA_PATH` when needed.
|
||||
|
||||
```sh
|
||||
export PLAYER_ORIGIN=http://192.168.1.20:3000
|
||||
./scripts/smoke-test.sh
|
||||
```
|
||||
|
||||
## Publishing Updates
|
||||
|
||||
Use a staging directory on the same filesystem as the media library, validate
|
||||
the staged MP4, and atomically rename it into place. Restart the container only
|
||||
after the rename completes:
|
||||
|
||||
```sh
|
||||
./scripts/validate-media.sh /srv/media/.staging
|
||||
mv -T /srv/media/.staging/example.mp4 /srv/media/movies/example.mp4
|
||||
./scripts/refresh-vod.sh
|
||||
```
|
||||
|
||||
The service sends `Cache-Control: no-store` and the restart clears VOD caches.
|
||||
This makes a replacement visible promptly, but intentionally interrupts active
|
||||
playback for a short period. Do not overwrite MP4 files in place.
|
||||
|
||||
## Operations
|
||||
|
||||
- Health endpoint: `GET /healthz`
|
||||
- Logs: `docker compose logs -f vod`
|
||||
- Stop: `docker compose down`
|
||||
- The host firewall should limit TCP `8124` to the trusted LAN. This service
|
||||
does not provide TLS or authentication.
|
||||
|
||||
The first build downloads the fixed source inputs and base image. The ARM64
|
||||
target host therefore needs access to Docker registries, `nginx.org`, and
|
||||
GitHub during `docker compose build`.
|
||||
@@ -0,0 +1 @@
|
||||
"""Media catalog service package."""
|
||||
+475
@@ -0,0 +1,475 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Prebuild and serve a read-only MP4 catalog snapshot."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import threading
|
||||
import uuid
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime
|
||||
from http import HTTPStatus
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote, urlsplit
|
||||
|
||||
|
||||
LOG = logging.getLogger(__name__)
|
||||
COVER_NAME = re.compile(r"^[0-9a-f]{64}\.jpg$")
|
||||
DEFAULT_REFRESH_INTERVAL_SECONDS = 6 * 60 * 60
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ProbeResult:
|
||||
duration: float
|
||||
attached_picture_stream: int | None
|
||||
|
||||
|
||||
def inspect_media(source: Path) -> ProbeResult | None:
|
||||
"""Validate a source file against the HLS service's supported streams."""
|
||||
command = [
|
||||
"ffprobe",
|
||||
"-v",
|
||||
"error",
|
||||
"-show_entries",
|
||||
"format=duration:stream=index,codec_name,codec_type:stream_disposition=attached_pic",
|
||||
"-of",
|
||||
"json",
|
||||
str(source),
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(command, capture_output=True, check=False, text=True, timeout=30)
|
||||
except (OSError, subprocess.TimeoutExpired) as error:
|
||||
LOG.warning("Unable to inspect %s: %s", source, error)
|
||||
return None
|
||||
|
||||
if result.returncode != 0:
|
||||
LOG.warning("Ignoring unreadable media %s: %s", source, result.stderr.strip())
|
||||
return None
|
||||
|
||||
try:
|
||||
payload = json.loads(result.stdout)
|
||||
duration = float(payload["format"]["duration"])
|
||||
except (KeyError, TypeError, ValueError, json.JSONDecodeError):
|
||||
LOG.warning("Ignoring media with incomplete metadata: %s", source)
|
||||
return None
|
||||
|
||||
video_count = 0
|
||||
audio_count = 0
|
||||
attached_picture_stream: int | None = None
|
||||
for stream in payload.get("streams", []):
|
||||
codec_type = stream.get("codec_type")
|
||||
codec_name = stream.get("codec_name")
|
||||
attached_picture = stream.get("disposition", {}).get("attached_pic") == 1
|
||||
if codec_type == "video" and attached_picture:
|
||||
attached_picture_stream = stream.get("index")
|
||||
elif codec_type == "video" and codec_name == "h264":
|
||||
video_count += 1
|
||||
elif codec_type == "audio" and codec_name == "aac":
|
||||
audio_count += 1
|
||||
else:
|
||||
LOG.warning("Ignoring unsupported media %s", source)
|
||||
return None
|
||||
|
||||
if video_count != 1 or audio_count != 1 or duration < 0:
|
||||
LOG.warning("Ignoring unsupported media %s", source)
|
||||
return None
|
||||
return ProbeResult(duration=duration, attached_picture_stream=attached_picture_stream)
|
||||
|
||||
|
||||
def playlist_url(relative_path: Path) -> str:
|
||||
encoded_path = "/".join(quote(part, safe="") for part in relative_path.parts)
|
||||
return f"/hls/{encoded_path}/master.m3u8"
|
||||
|
||||
|
||||
def cache_key(relative_path: Path, stat_result: os.stat_result) -> str:
|
||||
source_fingerprint = f"{relative_path.as_posix()}:{stat_result.st_size}:{stat_result.st_mtime_ns}"
|
||||
return hashlib.sha256(source_fingerprint.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
class Catalog:
|
||||
def __init__(
|
||||
self,
|
||||
media_dir: Path,
|
||||
cache_dir: Path,
|
||||
refresh_interval: float = DEFAULT_REFRESH_INTERVAL_SECONDS,
|
||||
auto_refresh: bool = True,
|
||||
) -> None:
|
||||
self.media_dir = media_dir.resolve()
|
||||
self.cache_dir = cache_dir.resolve()
|
||||
self.releases_dir = self.cache_dir / "releases"
|
||||
self.current_link = self.cache_dir / "current"
|
||||
self.releases_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
self._state_lock = threading.Lock()
|
||||
self._refresh_event = threading.Event()
|
||||
self._stop_event = threading.Event()
|
||||
self._build_active = False
|
||||
self._refresh_pending = False
|
||||
self._current_release: Path | None = None
|
||||
self._media_payload: bytes | None = None
|
||||
self._status: dict[str, object] = {
|
||||
"state": "building",
|
||||
"lastSuccessAt": None,
|
||||
"validFiles": 0,
|
||||
"skippedFiles": 0,
|
||||
"lastError": None,
|
||||
}
|
||||
self._load_current_snapshot()
|
||||
self._refresh_interval = refresh_interval
|
||||
self._worker: threading.Thread | None = None
|
||||
if auto_refresh:
|
||||
self._worker = threading.Thread(target=self._worker_loop, name="catalog-refresh", daemon=True)
|
||||
self._worker.start()
|
||||
self.request_rescan()
|
||||
|
||||
def ready(self) -> bool:
|
||||
with self._state_lock:
|
||||
return self._media_payload is not None and self._current_release is not None
|
||||
|
||||
def media_payload(self) -> bytes | None:
|
||||
with self._state_lock:
|
||||
return self._media_payload
|
||||
|
||||
def cover_payload(self, cover_name: str) -> bytes | None:
|
||||
if not COVER_NAME.fullmatch(cover_name):
|
||||
return None
|
||||
with self._state_lock:
|
||||
release = self._current_release
|
||||
if release is None:
|
||||
return None
|
||||
try:
|
||||
return (release / "covers" / cover_name).read_bytes()
|
||||
except OSError:
|
||||
return None
|
||||
|
||||
def status(self) -> dict[str, object]:
|
||||
with self._state_lock:
|
||||
return {"ready": self._media_payload is not None, **self._status}
|
||||
|
||||
def request_rescan(self) -> dict[str, object]:
|
||||
with self._state_lock:
|
||||
if not self._build_active and not self._refresh_pending:
|
||||
self._refresh_pending = True
|
||||
self._status["state"] = "building"
|
||||
self._status["lastError"] = None
|
||||
self._refresh_event.set()
|
||||
return {"ready": self._media_payload is not None, **self._status}
|
||||
|
||||
def refresh_now(self) -> None:
|
||||
"""Synchronously build a snapshot. Intended for tests and worker use."""
|
||||
with self._state_lock:
|
||||
self._build_active = True
|
||||
self._refresh_pending = False
|
||||
self._status["state"] = "building"
|
||||
self._status["lastError"] = None
|
||||
try:
|
||||
release, payload, metadata = self._build_snapshot()
|
||||
self._publish_snapshot(release, payload, metadata)
|
||||
except Exception as error: # Keep the previous release available on a failed build.
|
||||
LOG.exception("Catalog refresh failed")
|
||||
with self._state_lock:
|
||||
self._status["state"] = "ready" if self._media_payload is not None else "failed"
|
||||
self._status["lastError"] = str(error)
|
||||
finally:
|
||||
with self._state_lock:
|
||||
self._build_active = False
|
||||
|
||||
def _worker_loop(self) -> None:
|
||||
while not self._stop_event.is_set():
|
||||
requested = self._refresh_event.wait(self._refresh_interval)
|
||||
if self._stop_event.is_set():
|
||||
return
|
||||
self._refresh_event.clear()
|
||||
with self._state_lock:
|
||||
if not requested and not self._refresh_pending:
|
||||
self._refresh_pending = True
|
||||
self._status["state"] = "building"
|
||||
if self._build_active or not self._refresh_pending:
|
||||
continue
|
||||
self.refresh_now()
|
||||
|
||||
def _build_snapshot(self) -> tuple[Path, bytes, dict[str, object]]:
|
||||
release_id = f"{datetime.now(UTC).strftime('%Y%m%dT%H%M%SZ')}-{uuid.uuid4().hex}"
|
||||
staging = self.releases_dir / f".staging-{release_id}"
|
||||
release = self.releases_dir / release_id
|
||||
covers_dir = staging / "covers"
|
||||
covers_dir.mkdir(parents=True)
|
||||
valid_files = 0
|
||||
skipped_files = 0
|
||||
items: list[dict[str, str]] = []
|
||||
|
||||
try:
|
||||
for source in self._media_files():
|
||||
try:
|
||||
relative_path = source.relative_to(self.media_dir)
|
||||
before = source.stat()
|
||||
except OSError:
|
||||
skipped_files += 1
|
||||
continue
|
||||
|
||||
probe = inspect_media(source)
|
||||
if probe is None:
|
||||
skipped_files += 1
|
||||
continue
|
||||
|
||||
cover_name = f"{cache_key(relative_path, before)}.jpg"
|
||||
if not self._generate_cover(source, covers_dir / cover_name, probe):
|
||||
skipped_files += 1
|
||||
continue
|
||||
|
||||
try:
|
||||
after = source.stat()
|
||||
except OSError:
|
||||
skipped_files += 1
|
||||
continue
|
||||
if (before.st_size, before.st_mtime_ns) != (after.st_size, after.st_mtime_ns):
|
||||
LOG.warning("Ignoring media changed while processing: %s", source)
|
||||
(covers_dir / cover_name).unlink(missing_ok=True)
|
||||
skipped_files += 1
|
||||
continue
|
||||
|
||||
valid_files += 1
|
||||
items.append(
|
||||
{
|
||||
"path": relative_path.as_posix(),
|
||||
"name": source.stem,
|
||||
"playlistUrl": playlist_url(relative_path),
|
||||
"coverUrl": f"/api/media/covers/{cover_name}",
|
||||
}
|
||||
)
|
||||
|
||||
items.sort(key=lambda item: (item["name"].casefold(), item["path"].casefold()))
|
||||
payload = json.dumps(items, ensure_ascii=False, separators=(",", ":")).encode("utf-8")
|
||||
generated_at = datetime.now(UTC).isoformat().replace("+00:00", "Z")
|
||||
metadata: dict[str, object] = {
|
||||
"generatedAt": generated_at,
|
||||
"validFiles": valid_files,
|
||||
"skippedFiles": skipped_files,
|
||||
}
|
||||
(staging / "media.json").write_bytes(payload)
|
||||
(staging / "metadata.json").write_text(json.dumps(metadata, separators=(",", ":")), encoding="utf-8")
|
||||
staging.replace(release)
|
||||
return release, payload, metadata
|
||||
except Exception:
|
||||
shutil.rmtree(staging, ignore_errors=True)
|
||||
raise
|
||||
|
||||
def _publish_snapshot(self, release: Path, payload: bytes, metadata: dict[str, object]) -> None:
|
||||
pending_link = self.cache_dir / f".current-{uuid.uuid4().hex}"
|
||||
os.symlink(str(release.relative_to(self.cache_dir)), pending_link)
|
||||
try:
|
||||
os.replace(pending_link, self.current_link)
|
||||
finally:
|
||||
pending_link.unlink(missing_ok=True)
|
||||
|
||||
with self._state_lock:
|
||||
self._current_release = release
|
||||
self._media_payload = payload
|
||||
self._status.update(
|
||||
{
|
||||
"state": "ready",
|
||||
"lastSuccessAt": metadata["generatedAt"],
|
||||
"validFiles": metadata["validFiles"],
|
||||
"skippedFiles": metadata["skippedFiles"],
|
||||
"lastError": None,
|
||||
}
|
||||
)
|
||||
self._cleanup_old_releases(release)
|
||||
|
||||
def _load_current_snapshot(self) -> None:
|
||||
try:
|
||||
release = self.current_link.resolve(strict=True)
|
||||
payload = (release / "media.json").read_bytes()
|
||||
items = json.loads(payload)
|
||||
metadata = json.loads((release / "metadata.json").read_text(encoding="utf-8"))
|
||||
if not isinstance(items, list) or not isinstance(metadata, dict):
|
||||
raise ValueError("invalid snapshot")
|
||||
except (OSError, ValueError, json.JSONDecodeError) as error:
|
||||
if self.current_link.exists() or self.current_link.is_symlink():
|
||||
LOG.warning("Ignoring invalid catalog snapshot: %s", error)
|
||||
return
|
||||
|
||||
self._current_release = release
|
||||
self._media_payload = payload
|
||||
self._status.update(
|
||||
{
|
||||
"state": "ready",
|
||||
"lastSuccessAt": metadata.get("generatedAt"),
|
||||
"validFiles": metadata.get("validFiles", 0),
|
||||
"skippedFiles": metadata.get("skippedFiles", 0),
|
||||
}
|
||||
)
|
||||
|
||||
def _cleanup_old_releases(self, active_release: Path) -> None:
|
||||
releases = sorted(
|
||||
(path for path in self.releases_dir.iterdir() if path.is_dir() and not path.name.startswith(".staging-")),
|
||||
key=lambda path: path.stat().st_mtime_ns,
|
||||
reverse=True,
|
||||
)
|
||||
keep = {active_release, *releases[:2]}
|
||||
for release in releases:
|
||||
if release not in keep:
|
||||
shutil.rmtree(release, ignore_errors=True)
|
||||
|
||||
def _generate_cover(self, source: Path, cover_path: Path, probe: ProbeResult) -> bool:
|
||||
temporary_path = cover_path.with_name(f".{cover_path.stem}.{os.getpid()}.jpg")
|
||||
if probe.attached_picture_stream is not None:
|
||||
command = [
|
||||
"ffmpeg",
|
||||
"-nostdin",
|
||||
"-v",
|
||||
"error",
|
||||
"-i",
|
||||
str(source),
|
||||
"-map",
|
||||
f"0:{probe.attached_picture_stream}",
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-vf",
|
||||
"scale=480:-2",
|
||||
"-q:v",
|
||||
"2",
|
||||
"-y",
|
||||
str(temporary_path),
|
||||
]
|
||||
else:
|
||||
timestamp = min(10.0, probe.duration * 0.1)
|
||||
command = [
|
||||
"ffmpeg",
|
||||
"-nostdin",
|
||||
"-v",
|
||||
"error",
|
||||
"-ss",
|
||||
f"{timestamp:.3f}",
|
||||
"-i",
|
||||
str(source),
|
||||
"-frames:v",
|
||||
"1",
|
||||
"-vf",
|
||||
"scale=480:-2",
|
||||
"-q:v",
|
||||
"2",
|
||||
"-y",
|
||||
str(temporary_path),
|
||||
]
|
||||
try:
|
||||
result = subprocess.run(command, capture_output=True, check=False, text=True, timeout=60)
|
||||
if result.returncode != 0 or not temporary_path.is_file() or temporary_path.stat().st_size == 0:
|
||||
LOG.warning("Unable to generate cover for %s: %s", source, result.stderr.strip())
|
||||
return False
|
||||
temporary_path.replace(cover_path)
|
||||
return True
|
||||
except (OSError, subprocess.TimeoutExpired) as error:
|
||||
LOG.warning("Unable to generate cover for %s: %s", source, error)
|
||||
return False
|
||||
finally:
|
||||
temporary_path.unlink(missing_ok=True)
|
||||
|
||||
def _media_files(self) -> list[Path]:
|
||||
files: list[Path] = []
|
||||
for root, directories, filenames in os.walk(self.media_dir, followlinks=False):
|
||||
directories[:] = [name for name in directories if not (Path(root) / name).is_symlink()]
|
||||
for filename in filenames:
|
||||
source = Path(root) / filename
|
||||
if source.suffix.lower() == ".mp4" and source.is_file() and not source.is_symlink():
|
||||
files.append(source)
|
||||
return sorted(files, key=lambda path: path.relative_to(self.media_dir).as_posix().casefold())
|
||||
|
||||
|
||||
class CatalogRequestHandler(BaseHTTPRequestHandler):
|
||||
server: "CatalogHTTPServer"
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802
|
||||
request = urlsplit(self.path)
|
||||
if request.path == "/healthz":
|
||||
if self.server.catalog.ready():
|
||||
self._send_bytes(HTTPStatus.OK, b"ok\n", "text/plain; charset=utf-8")
|
||||
else:
|
||||
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"error": "catalog is building"})
|
||||
elif request.path == "/api/media/status":
|
||||
self._send_json(HTTPStatus.OK, self.server.catalog.status())
|
||||
elif request.path == "/api/media":
|
||||
payload = self.server.catalog.media_payload()
|
||||
if payload is None:
|
||||
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"error": "catalog is building"})
|
||||
else:
|
||||
self._send_bytes(HTTPStatus.OK, payload, "application/json; charset=utf-8")
|
||||
elif request.path.startswith("/api/media/covers/"):
|
||||
if not self.server.catalog.ready():
|
||||
self._send_json(HTTPStatus.SERVICE_UNAVAILABLE, {"error": "catalog is building"})
|
||||
else:
|
||||
self._send_cover(request.path.rsplit("/", 1)[-1])
|
||||
else:
|
||||
self.send_error(HTTPStatus.NOT_FOUND)
|
||||
|
||||
def do_HEAD(self) -> None: # noqa: N802
|
||||
self.do_GET()
|
||||
|
||||
def do_POST(self) -> None: # noqa: N802
|
||||
if urlsplit(self.path).path != "/api/media/rescan":
|
||||
self.send_error(HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
self._send_json(HTTPStatus.ACCEPTED, self.server.catalog.request_rescan())
|
||||
|
||||
def do_OPTIONS(self) -> None: # noqa: N802
|
||||
self.send_response(HTTPStatus.NO_CONTENT)
|
||||
self.send_header("Allow", "GET, HEAD, POST, OPTIONS")
|
||||
self.end_headers()
|
||||
|
||||
def _send_cover(self, cover_name: str) -> None:
|
||||
payload = self.server.catalog.cover_payload(cover_name)
|
||||
if payload is None:
|
||||
self.send_error(HTTPStatus.NOT_FOUND)
|
||||
return
|
||||
self._send_bytes(HTTPStatus.OK, payload, "image/jpeg")
|
||||
|
||||
def _send_json(self, status: HTTPStatus, payload: object) -> None:
|
||||
self._send_bytes(
|
||||
status,
|
||||
json.dumps(payload, ensure_ascii=False, separators=(",", ":")).encode("utf-8"),
|
||||
"application/json; charset=utf-8",
|
||||
)
|
||||
|
||||
def _send_bytes(self, status: HTTPStatus, body: bytes, content_type: str) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Cache-Control", "no-store")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, format_string: str, *args: object) -> None:
|
||||
LOG.info("%s - %s", self.address_string(), format_string % args)
|
||||
|
||||
|
||||
class CatalogHTTPServer(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, address: tuple[str, int], catalog: Catalog) -> None:
|
||||
super().__init__(address, CatalogRequestHandler)
|
||||
self.catalog = catalog
|
||||
|
||||
|
||||
def main() -> None:
|
||||
logging.basicConfig(level=os.environ.get("LOG_LEVEL", "INFO"), format="%(asctime)s %(levelname)s %(message)s")
|
||||
media_dir = Path(os.environ.get("MEDIA_DIR", "/media"))
|
||||
cache_dir = Path(os.environ.get("CATALOG_CACHE_DIR", "/cache"))
|
||||
refresh_interval = float(os.environ.get("CATALOG_REFRESH_INTERVAL_SECONDS", str(DEFAULT_REFRESH_INTERVAL_SECONDS)))
|
||||
port = int(os.environ.get("CATALOG_PORT", "8081"))
|
||||
if not media_dir.is_dir():
|
||||
raise SystemExit(f"Media directory does not exist: {media_dir}")
|
||||
if refresh_interval <= 0:
|
||||
raise SystemExit("CATALOG_REFRESH_INTERVAL_SECONDS must be positive")
|
||||
CatalogHTTPServer(("0.0.0.0", port), Catalog(media_dir, cache_dir, refresh_interval)).serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,69 @@
|
||||
services:
|
||||
vod:
|
||||
build:
|
||||
context: .
|
||||
target: vod
|
||||
args:
|
||||
NGINX_VERSION: "${NGINX_VERSION:-1.30.5}"
|
||||
KALTURA_VOD_REF: "${KALTURA_VOD_REF:-26f06877b0f2a2336e59cda93a3de18d7b23a3e2}"
|
||||
image: "${IMAGE_NAME:-nginx-hls-vod}:local"
|
||||
environment:
|
||||
PLAYER_ORIGIN: "${PLAYER_ORIGIN:?Set PLAYER_ORIGIN in .env}"
|
||||
depends_on:
|
||||
catalog:
|
||||
condition: service_healthy
|
||||
ports:
|
||||
- "${HLS_BIND_ADDRESS:-0.0.0.0}:${HLS_PORT:-8124}:8080"
|
||||
volumes:
|
||||
- type: bind
|
||||
source: "${MEDIA_DIR:?Set MEDIA_DIR in .env}"
|
||||
target: /media
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: false
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,nosuid,nodev,noexec,size=64m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 128
|
||||
mem_limit: 512m
|
||||
cpus: 2.0
|
||||
|
||||
catalog:
|
||||
build:
|
||||
context: .
|
||||
target: catalog
|
||||
image: "${CATALOG_IMAGE:-nginx-hls-catalog}:local"
|
||||
user: "0:0"
|
||||
environment:
|
||||
MEDIA_DIR: /media
|
||||
CATALOG_CACHE_DIR: /cache
|
||||
CATALOG_REFRESH_INTERVAL_SECONDS: "${CATALOG_REFRESH_INTERVAL_SECONDS:-21600}"
|
||||
CATALOG_PORT: "8081"
|
||||
volumes:
|
||||
- type: bind
|
||||
source: "${MEDIA_DIR:?Set MEDIA_DIR in .env}"
|
||||
target: /media
|
||||
read_only: true
|
||||
bind:
|
||||
create_host_path: false
|
||||
- type: bind
|
||||
source: "${CATALOG_CACHE_DIR:?Set CATALOG_CACHE_DIR in .env}"
|
||||
target: /cache
|
||||
bind:
|
||||
create_host_path: false
|
||||
restart: unless-stopped
|
||||
read_only: true
|
||||
tmpfs:
|
||||
- /tmp:rw,nosuid,nodev,noexec,size=64m
|
||||
cap_drop:
|
||||
- ALL
|
||||
security_opt:
|
||||
- no-new-privileges:true
|
||||
pids_limit: 128
|
||||
mem_limit: 512m
|
||||
cpus: 2.0
|
||||
@@ -0,0 +1,195 @@
|
||||
worker_processes auto;
|
||||
error_log /dev/stderr warn;
|
||||
pid /tmp/nginx.pid;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
}
|
||||
|
||||
thread_pool vod_io threads=2 max_queue=65536;
|
||||
|
||||
http {
|
||||
include /etc/nginx/mime.types;
|
||||
default_type application/octet-stream;
|
||||
|
||||
log_format vod '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent" rt=$request_time';
|
||||
access_log /dev/stdout vod;
|
||||
|
||||
sendfile on;
|
||||
tcp_nopush on;
|
||||
keepalive_timeout 65;
|
||||
server_tokens off;
|
||||
|
||||
aio threads=vod_io;
|
||||
|
||||
upstream catalog_backend {
|
||||
server catalog:8081;
|
||||
}
|
||||
|
||||
server {
|
||||
listen 8080;
|
||||
server_name _;
|
||||
autoindex off;
|
||||
|
||||
vod_mode local;
|
||||
vod_metadata_cache metadata_cache 64m;
|
||||
vod_response_cache response_cache 16m;
|
||||
vod_open_file_thread_pool vod_io;
|
||||
|
||||
open_file_cache max=1000 inactive=30s;
|
||||
open_file_cache_valid 30s;
|
||||
open_file_cache_min_uses 1;
|
||||
open_file_cache_errors on;
|
||||
|
||||
location = /healthz {
|
||||
access_log off;
|
||||
default_type text/plain;
|
||||
return 200 "ok\n";
|
||||
}
|
||||
|
||||
location ^~ /hls/ {
|
||||
alias /media/;
|
||||
vod hls;
|
||||
vod_segment_duration 6000;
|
||||
vod_align_segments_to_key_frames on;
|
||||
vod_manifest_segment_durations_mode accurate;
|
||||
|
||||
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||
add_header Access-Control-Expose-Headers "Accept-Ranges, Content-Length, Content-Range, Content-Type" always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
add_header Vary "Origin" always;
|
||||
|
||||
if ($request_method = OPTIONS) {
|
||||
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||
add_header Access-Control-Max-Age 600 always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
add_header Vary "Origin" always;
|
||||
return 204;
|
||||
}
|
||||
|
||||
if ($request_method !~ ^(GET|HEAD)$) {
|
||||
return 405;
|
||||
}
|
||||
}
|
||||
|
||||
location = /api/media {
|
||||
proxy_pass http://catalog_backend;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
proxy_hide_header Cache-Control;
|
||||
|
||||
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
add_header Vary "Origin" always;
|
||||
|
||||
if ($request_method = OPTIONS) {
|
||||
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||
add_header Access-Control-Max-Age 600 always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
add_header Vary "Origin" always;
|
||||
return 204;
|
||||
}
|
||||
|
||||
if ($request_method !~ ^(GET|HEAD)$) {
|
||||
return 405;
|
||||
}
|
||||
}
|
||||
|
||||
location = /api/media/status {
|
||||
proxy_pass http://catalog_backend;
|
||||
proxy_set_header Host $host;
|
||||
proxy_hide_header Cache-Control;
|
||||
|
||||
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
add_header Vary "Origin" always;
|
||||
|
||||
if ($request_method = OPTIONS) {
|
||||
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||
add_header Access-Control-Max-Age 600 always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
add_header Vary "Origin" always;
|
||||
return 204;
|
||||
}
|
||||
|
||||
if ($request_method !~ ^(GET|HEAD)$) {
|
||||
return 405;
|
||||
}
|
||||
}
|
||||
|
||||
location = /api/media/rescan {
|
||||
proxy_pass http://catalog_backend;
|
||||
proxy_set_header Host $host;
|
||||
proxy_hide_header Cache-Control;
|
||||
|
||||
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||
add_header Access-Control-Allow-Methods "POST, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
add_header Vary "Origin" always;
|
||||
|
||||
if ($request_method = OPTIONS) {
|
||||
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||
add_header Access-Control-Allow-Methods "POST, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||
add_header Access-Control-Max-Age 600 always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
add_header Vary "Origin" always;
|
||||
return 204;
|
||||
}
|
||||
|
||||
if ($request_method != POST) {
|
||||
return 405;
|
||||
}
|
||||
}
|
||||
|
||||
location ^~ /api/media/covers/ {
|
||||
proxy_pass http://catalog_backend;
|
||||
proxy_set_header Host $host;
|
||||
proxy_hide_header Cache-Control;
|
||||
|
||||
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||
add_header Access-Control-Expose-Headers "Content-Length, Content-Type" always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
add_header Vary "Origin" always;
|
||||
|
||||
if ($request_method = OPTIONS) {
|
||||
add_header Access-Control-Allow-Origin "${PLAYER_ORIGIN}" always;
|
||||
add_header Access-Control-Allow-Methods "GET, HEAD, OPTIONS" always;
|
||||
add_header Access-Control-Allow-Headers "Origin, Range, Accept, Content-Type" always;
|
||||
add_header Access-Control-Max-Age 600 always;
|
||||
add_header Cache-Control "no-store" always;
|
||||
add_header Vary "Origin" always;
|
||||
return 204;
|
||||
}
|
||||
|
||||
if ($request_method !~ ^(GET|HEAD)$) {
|
||||
return 405;
|
||||
}
|
||||
}
|
||||
|
||||
location / {
|
||||
return 404;
|
||||
}
|
||||
}
|
||||
}
|
||||
Executable
+17
@@ -0,0 +1,17 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
: "${PLAYER_ORIGIN:?PLAYER_ORIGIN must be set}"
|
||||
|
||||
if ! printf '%s' "$PLAYER_ORIGIN" | grep -Eq '^https?://[A-Za-z0-9._:-]+$'; then
|
||||
echo "PLAYER_ORIGIN must be an origin such as http://192.168.1.20:3000" >&2
|
||||
exit 64
|
||||
fi
|
||||
|
||||
envsubst '${PLAYER_ORIGIN}' \
|
||||
< /etc/nginx/templates/nginx.conf.template \
|
||||
> /tmp/nginx.conf
|
||||
|
||||
nginx -t -c /tmp/nginx.conf
|
||||
exec nginx -c /tmp/nginx.conf -g 'daemon off;'
|
||||
|
||||
Executable
+6
@@ -0,0 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
echo "Restarting VOD after an already-completed atomic media update."
|
||||
docker compose restart vod
|
||||
|
||||
Executable
+52
@@ -0,0 +1,52 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
: "${PLAYER_ORIGIN:?Set PLAYER_ORIGIN before running this test}"
|
||||
|
||||
base_url=${HLS_BASE_URL:-http://127.0.0.1:8124}
|
||||
media_path=${TEST_MEDIA_PATH:-aa/1170193193-1-192.mp4}
|
||||
tmpdir=$(mktemp -d)
|
||||
trap 'rm -rf "$tmpdir"' EXIT
|
||||
|
||||
resolve_url() {
|
||||
local base=$1
|
||||
local reference=$2
|
||||
|
||||
case "$reference" in
|
||||
http://*|https://*) printf '%s\n' "$reference" ;;
|
||||
/*) printf '%s%s\n' "$(printf '%s\n' "$base" | sed -E 's#(https?://[^/]+).*#\1#')" "$reference" ;;
|
||||
*) printf '%s/%s\n' "${base%/*}" "$reference" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
master_url="${base_url%/}/hls/${media_path}/master.m3u8"
|
||||
master_headers="$tmpdir/master.headers"
|
||||
master_playlist="$tmpdir/master.m3u8"
|
||||
|
||||
curl --fail --silent --show-error --dump-header "$master_headers" --output "$master_playlist" "$master_url"
|
||||
tr -d '\r' < "$master_headers" | grep -Fxi "Access-Control-Allow-Origin: $PLAYER_ORIGIN" >/dev/null
|
||||
tr -d '\r' < "$master_headers" | grep -Fxi 'Cache-Control: no-store' >/dev/null
|
||||
grep -Fqx '#EXTM3U' "$master_playlist" >/dev/null
|
||||
|
||||
child_reference=$(awk 'found { print; exit } /^#EXT-X-STREAM-INF/ { found=1 }' "$master_playlist")
|
||||
if [[ -z $child_reference ]]; then
|
||||
child_url=$master_url
|
||||
else
|
||||
child_url=$(resolve_url "$master_url" "$child_reference")
|
||||
fi
|
||||
|
||||
child_playlist="$tmpdir/index.m3u8"
|
||||
curl --fail --silent --show-error --output "$child_playlist" "$child_url"
|
||||
grep -Fqx '#EXTM3U' "$child_playlist" >/dev/null
|
||||
|
||||
segment_reference=$(awk '!/^#/ && NF { print; exit }' "$child_playlist")
|
||||
if [[ -z $segment_reference ]]; then
|
||||
echo "No HLS segment was found in $child_url" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
segment_url=$(resolve_url "$child_url" "$segment_reference")
|
||||
curl --fail --silent --show-error --output /dev/null "$segment_url"
|
||||
|
||||
printf 'HLS smoke test passed: %s\n' "$master_url"
|
||||
|
||||
Executable
+100
@@ -0,0 +1,100 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
media_root=${1:-data}
|
||||
|
||||
if ! command -v ffprobe >/dev/null 2>&1; then
|
||||
echo "ffprobe is required to validate media." >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
if [[ ! -d $media_root ]]; then
|
||||
printf 'Media directory does not exist: %s\n' "$media_root" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
valid_count=0
|
||||
invalid_count=0
|
||||
|
||||
validate_file() {
|
||||
local file=$1
|
||||
local format_name streams codec type attached_pic
|
||||
local video_count=0
|
||||
local audio_count=0
|
||||
local invalid_reason=
|
||||
|
||||
if ! format_name=$(ffprobe -v error -show_entries format=format_name \
|
||||
-of default=noprint_wrappers=1:nokey=1 "$file"); then
|
||||
printf 'INVALID: %q: unreadable or corrupt MP4\n' "$file" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if [[ ,$format_name, != *,mp4,* ]]; then
|
||||
printf 'INVALID: %q: expected an MP4 container, got %s\n' "$file" "$format_name" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ! streams=$(ffprobe -v error \
|
||||
-show_entries stream=codec_name,codec_type:stream_disposition=attached_pic \
|
||||
-of csv=p=0 "$file"); then
|
||||
printf 'INVALID: %q: unable to inspect streams\n' "$file" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
while IFS=, read -r codec type attached_pic; do
|
||||
case "$type" in
|
||||
video)
|
||||
if [[ $attached_pic == 1 ]]; then
|
||||
continue
|
||||
fi
|
||||
if [[ $codec != h264 ]]; then
|
||||
invalid_reason="unsupported video codec $codec"
|
||||
fi
|
||||
((video_count += 1))
|
||||
;;
|
||||
audio)
|
||||
if [[ $codec != aac ]]; then
|
||||
invalid_reason="unsupported audio codec $codec"
|
||||
fi
|
||||
((audio_count += 1))
|
||||
;;
|
||||
*)
|
||||
invalid_reason="unsupported $type track ($codec)"
|
||||
;;
|
||||
esac
|
||||
done <<< "$streams"
|
||||
|
||||
if [[ -n $invalid_reason ]]; then
|
||||
printf 'INVALID: %q: %s\n' "$file" "$invalid_reason" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
if ((video_count != 1 || audio_count != 1)); then
|
||||
printf 'INVALID: %q: expected one H.264 video and one AAC audio track; found %d video and %d audio\n' \
|
||||
"$file" "$video_count" "$audio_count" >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
printf 'VALID: %s\n' "$file"
|
||||
}
|
||||
|
||||
while IFS= read -r -d '' file; do
|
||||
if validate_file "$file"; then
|
||||
((valid_count += 1))
|
||||
else
|
||||
((invalid_count += 1))
|
||||
fi
|
||||
done < <(find "$media_root" -type f -iname '*.mp4' -print0)
|
||||
|
||||
if ((valid_count == 0)); then
|
||||
echo "No valid MP4 files were found." >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ((invalid_count > 0)); then
|
||||
printf 'Validation failed: %d valid, %d invalid file(s).\n' "$valid_count" "$invalid_count" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
printf 'Validation passed: %d MP4 file(s).\n' "$valid_count"
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
import json
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from catalog.app import Catalog, ProbeResult, cache_key, inspect_media, playlist_url
|
||||
|
||||
|
||||
class CatalogTests(unittest.TestCase):
|
||||
def test_playlist_url_encodes_each_path_segment(self) -> None:
|
||||
self.assertEqual(
|
||||
playlist_url(Path("中文 media") / "clip #1.mp4"),
|
||||
"/hls/%E4%B8%AD%E6%96%87%20media/clip%20%231.mp4/master.m3u8",
|
||||
)
|
||||
|
||||
def test_cache_key_changes_when_media_changes(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
source = Path(temporary_directory) / "clip.mp4"
|
||||
source.write_bytes(b"a")
|
||||
first = cache_key(Path("clip.mp4"), source.stat())
|
||||
source.write_bytes(b"updated")
|
||||
second = cache_key(Path("clip.mp4"), source.stat())
|
||||
self.assertNotEqual(first, second)
|
||||
|
||||
def test_refresh_publishes_validated_list_and_cached_cover(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
media_dir = root / "media"
|
||||
cache_dir = root / "cache"
|
||||
media_dir.mkdir()
|
||||
source = media_dir / "clip.mp4"
|
||||
source.write_bytes(b"video")
|
||||
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
|
||||
probe_payload = json.dumps(
|
||||
{
|
||||
"format": {"duration": "20"},
|
||||
"streams": [
|
||||
{"index": 0, "codec_type": "video", "codec_name": "h264", "disposition": {"attached_pic": 0}},
|
||||
{"index": 1, "codec_type": "audio", "codec_name": "aac", "disposition": {"attached_pic": 0}},
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
def fake_run(command: list[str], **_kwargs: object) -> subprocess.CompletedProcess[str]:
|
||||
if command[0] == "ffprobe":
|
||||
return subprocess.CompletedProcess(command, 0, probe_payload, "")
|
||||
Path(command[-1]).write_bytes(b"jpeg")
|
||||
return subprocess.CompletedProcess(command, 0, "", "")
|
||||
|
||||
self.assertFalse(catalog.ready())
|
||||
with patch("catalog.app.subprocess.run", side_effect=fake_run) as run:
|
||||
catalog.refresh_now()
|
||||
|
||||
self.assertTrue(catalog.ready())
|
||||
self.assertTrue((cache_dir / "current").is_symlink())
|
||||
items = json.loads(catalog.media_payload())
|
||||
self.assertEqual(len(items), 1)
|
||||
self.assertEqual(items[0]["path"], "clip.mp4")
|
||||
self.assertEqual(items[0]["playlistUrl"], "/hls/clip.mp4/master.m3u8")
|
||||
cover_name = items[0]["coverUrl"].rsplit("/", 1)[-1]
|
||||
self.assertEqual(catalog.cover_payload(cover_name), b"jpeg")
|
||||
self.assertEqual(run.call_count, 2)
|
||||
|
||||
with patch("catalog.app.subprocess.run") as run:
|
||||
self.assertEqual(json.loads(catalog.media_payload()), items)
|
||||
self.assertEqual(catalog.cover_payload(cover_name), b"jpeg")
|
||||
run.assert_not_called()
|
||||
|
||||
def test_refresh_excludes_invalid_media(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
media_dir = root / "media"
|
||||
cache_dir = root / "cache"
|
||||
media_dir.mkdir()
|
||||
(media_dir / "valid.mp4").write_bytes(b"valid")
|
||||
(media_dir / "broken.mp4").write_bytes(b"broken")
|
||||
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
|
||||
|
||||
def fake_probe(source: Path) -> ProbeResult | None:
|
||||
return ProbeResult(10.0, None) if source.name == "valid.mp4" else None
|
||||
|
||||
with patch("catalog.app.inspect_media", side_effect=fake_probe), patch.object(
|
||||
Catalog, "_generate_cover", return_value=True
|
||||
):
|
||||
catalog.refresh_now()
|
||||
|
||||
items = json.loads(catalog.media_payload())
|
||||
self.assertEqual([item["path"] for item in items], ["valid.mp4"])
|
||||
status = catalog.status()
|
||||
self.assertEqual(status["validFiles"], 1)
|
||||
self.assertEqual(status["skippedFiles"], 1)
|
||||
|
||||
def test_failed_refresh_keeps_previous_snapshot(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
media_dir = root / "media"
|
||||
cache_dir = root / "cache"
|
||||
media_dir.mkdir()
|
||||
(media_dir / "clip.mp4").write_bytes(b"video")
|
||||
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
|
||||
|
||||
with patch("catalog.app.inspect_media", return_value=ProbeResult(10.0, None)), patch.object(
|
||||
Catalog, "_generate_cover", return_value=True
|
||||
):
|
||||
catalog.refresh_now()
|
||||
previous_payload = catalog.media_payload()
|
||||
|
||||
with patch.object(Catalog, "_build_snapshot", side_effect=RuntimeError("disk full")):
|
||||
catalog.refresh_now()
|
||||
|
||||
self.assertTrue(catalog.ready())
|
||||
self.assertEqual(catalog.media_payload(), previous_payload)
|
||||
self.assertEqual(catalog.status()["state"], "ready")
|
||||
self.assertEqual(catalog.status()["lastError"], "disk full")
|
||||
|
||||
def test_existing_snapshot_is_loaded_without_subprocesses(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temporary_directory:
|
||||
root = Path(temporary_directory)
|
||||
media_dir = root / "media"
|
||||
cache_dir = root / "cache"
|
||||
media_dir.mkdir()
|
||||
(media_dir / "clip.mp4").write_bytes(b"video")
|
||||
builder = Catalog(media_dir, cache_dir, auto_refresh=False)
|
||||
with patch("catalog.app.inspect_media", return_value=ProbeResult(10.0, None)), patch.object(
|
||||
Catalog, "_generate_cover", return_value=True
|
||||
):
|
||||
builder.refresh_now()
|
||||
|
||||
with patch("catalog.app.subprocess.run") as run:
|
||||
catalog = Catalog(media_dir, cache_dir, auto_refresh=False)
|
||||
payload = catalog.media_payload()
|
||||
run.assert_not_called()
|
||||
self.assertTrue(catalog.ready())
|
||||
self.assertEqual(len(json.loads(payload)), 1)
|
||||
|
||||
def test_inspect_media_rejects_unsupported_streams(self) -> None:
|
||||
payload = {
|
||||
"format": {"duration": "12.5"},
|
||||
"streams": [
|
||||
{"index": 0, "codec_type": "video", "codec_name": "h264", "disposition": {"attached_pic": 0}},
|
||||
{"index": 1, "codec_type": "audio", "codec_name": "aac", "disposition": {"attached_pic": 0}},
|
||||
{"index": 2, "codec_type": "subtitle", "codec_name": "mov_text", "disposition": {"attached_pic": 0}},
|
||||
],
|
||||
}
|
||||
completed = subprocess.CompletedProcess([], 0, json.dumps(payload), "")
|
||||
with patch("catalog.app.subprocess.run", return_value=completed):
|
||||
self.assertIsNone(inspect_media(Path("clip.mp4")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user