commit 9dfc684869cae5ad1b5f4d1cfa52c04b42d1f64c Author: root Date: Sat Sep 26 13:55:43 2026 +0800 init diff --git a/N_m3u8DL_py/.gitignore b/N_m3u8DL_py/.gitignore new file mode 100644 index 0000000..1bdc0eb --- /dev/null +++ b/N_m3u8DL_py/.gitignore @@ -0,0 +1,6 @@ +.venv/ +.venv_network_failed/ +__pycache__/ +.pytest_cache/ +*.pyc +*.egg-info/ diff --git a/N_m3u8DL_py/README.md b/N_m3u8DL_py/README.md new file mode 100644 index 0000000..b6749d5 --- /dev/null +++ b/N_m3u8DL_py/README.md @@ -0,0 +1,90 @@ +# N_m3u8DL-PY + +`N_m3u8DL-PY` is a Python reimplementation of the downloader architecture in +the parent `N_m3u8DL-RE` project. It accepts HLS, DASH and Smooth Streaming +manifests from HTTP URLs or local files and downloads selected media tracks. + +```bash +python -m n_m3u8dl_py --auto-select +``` + +The command line intentionally follows the original project where practical. +Implemented workflows include: + +- HLS master/media playlists, `EXT-X-MAP`, byte ranges, discontinuities and + AES-128/AES-128-ECB decryption through the system `openssl` command. +- DASH `SegmentTemplate`, `SegmentTimeline`, `SegmentList`, `SegmentBase`, + BaseURL inheritance and common CENC metadata detection. +- Smooth Streaming manifest parsing for supported video/audio/text tracks. +- Parallel segment download, retries, HTTP headers/proxy support, range and + ad filtering, binary concat, ffmpeg merge and post-download muxing. + +DRM/CENC media cannot be decrypted in Python. The CLI passes supplied keys to +the configured external decryption tool (`mp4decrypt`, `shaka-packager`, or +`ffmpeg`) after download where possible. + +## Python API + +The package exports a stable synchronous API for integration in another Python +application. It never opens an interactive track picker. + +```python +from pathlib import Path + +from n_m3u8dl_py import DownloadRequest, N_m3u8DL, TrackSelection + +client = N_m3u8DL() +media = client.inspect("https://example.com/master.m3u8") + +# The default request chooses the best video, the best audio per language, +# and all subtitle tracks. Track IDs can be selected explicitly when needed. +result = client.download( + media, + DownloadRequest(output_dir=Path("downloads"), file_name="episode"), +) + +# Or use the one-call convenience method when track inspection is unnecessary. +result = client.download_url( + "https://example.com/master.m3u8", + DownloadRequest(output_dir=Path("downloads")), +) + +video = next(track for track in media.tracks if track.resolution) +video_only = client.download( + media, + DownloadRequest( + output_dir=Path("downloads"), + selection=TrackSelection(track_ids=(video.track_id,)), + ), +) +``` + +Pass `on_event` to `inspect()` or `download()` for structured progress. A +callback may return `False` to cancel the current task; cancellation raises +`DownloadCancelledError`. Successful calls return `MediaInfo` and +`DownloadResult`; manifest, selection, and download failures raise the +corresponding public exception classes. + +```python +from n_m3u8dl_py import DownloadEventKind + +def show_progress(event): + if event.kind is DownloadEventKind.SEGMENT_COMPLETED: + print(event.track_id, event.completed_segments, event.total_segments) + +media = client.inspect("playlist.m3u8", on_event=show_progress) +result = client.download(media, on_event=show_progress) +``` + +Create an editable local installation: + +```bash +python -m venv --system-site-packages .venv +.venv/bin/python -m pip install --no-build-isolation -e . +``` + +Run the test suite when `pytest` is available: + +```bash +.venv/bin/python -m pytest tests +``` diff --git a/N_m3u8DL_py/pyproject.toml b/N_m3u8DL_py/pyproject.toml new file mode 100644 index 0000000..2b76d06 --- /dev/null +++ b/N_m3u8DL_py/pyproject.toml @@ -0,0 +1,24 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "n-m3u8dl-py" +version = "0.2.0" +description = "Python implementation of the N_m3u8DL-RE DASH/HLS/MSS downloader" +requires-python = ">=3.10" +dependencies = [] + +[project.scripts] +N_m3u8DL-PY = "n_m3u8dl_py.cli:main" +n-m3u8dl-py = "n_m3u8dl_py.cli:main" + +[tool.setuptools] +package-dir = {"" = "src"} + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/__init__.py b/N_m3u8DL_py/src/n_m3u8dl_py/__init__.py new file mode 100644 index 0000000..6744a26 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/__init__.py @@ -0,0 +1,41 @@ +"""N_m3u8DL-PY public package API.""" + +from .api import ( + DownloadRequest, + DownloadResult, + DownloadedFile, + MediaInfo, + MediaTrack, + MuxRequest, + MuxResult, + N_m3u8DL, + RequestOptions, + TrackSelection, +) +from .errors import DownloadCancelledError, DownloadError, ManifestError, N_m3u8DLError, SelectionError +from .events import DownloadEvent, DownloadEventKind +from .models import EncryptMethod, MediaType + +__version__ = "0.2.0" + +__all__ = [ + "DownloadCancelledError", + "DownloadError", + "DownloadEvent", + "DownloadEventKind", + "DownloadRequest", + "DownloadResult", + "DownloadedFile", + "EncryptMethod", + "ManifestError", + "MediaInfo", + "MediaTrack", + "MediaType", + "MuxRequest", + "MuxResult", + "N_m3u8DL", + "N_m3u8DLError", + "RequestOptions", + "SelectionError", + "TrackSelection", +] diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/__main__.py b/N_m3u8DL_py/src/n_m3u8dl_py/__main__.py new file mode 100644 index 0000000..a049ad7 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/__main__.py @@ -0,0 +1,5 @@ +from .cli import main + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/api.py b/N_m3u8DL_py/src/n_m3u8dl_py/api.py new file mode 100644 index 0000000..3df0817 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/api.py @@ -0,0 +1,392 @@ +"""Stable, synchronous API for integrating N_m3u8DL-PY in Python programs.""" + +from __future__ import annotations + +import os +import shutil +from dataclasses import dataclass, field +from pathlib import Path +from types import MappingProxyType +from typing import Mapping + +from .downloader import DownloadManager, DownloadOptions, mux_outputs +from .errors import DownloadCancelledError, DownloadError, ManifestError, N_m3u8DLError, SelectionError +from .events import DownloadEvent, DownloadEventKind, EventCallback +from .extractor import ExtractedSource, load_source +from .http import HttpClient +from .models import EncryptMethod, MediaType, StreamSpec +from .selection import apply_custom_range, automatic_selection, clean_ads +from .utils import inferred_name, valid_filename + + +@dataclass(frozen=True) +class RequestOptions: + """Network and manifest parsing options used by :meth:`N_m3u8DL.inspect`.""" + + headers: Mapping[str, str] = field(default_factory=dict) + timeout: float = 100.0 + proxy: str | None = None + use_system_proxy: bool = True + base_url: str | None = None + append_url_params: bool = False + custom_hls_method: EncryptMethod | None = None + custom_hls_key: bytes | None = None + custom_hls_iv: bytes | None = None + allow_hls_multi_ext_map: bool = False + + def __post_init__(self) -> None: + if self.timeout <= 0: + raise ValueError("timeout must be greater than zero") + object.__setattr__(self, "headers", MappingProxyType(dict(self.headers))) + + +@dataclass(frozen=True) +class TrackSelection: + """Select explicit tracks or use automatic best-track selection.""" + + track_ids: tuple[str, ...] | None = None + auto_select: bool = True + sub_only: bool = False + + def __post_init__(self) -> None: + if self.track_ids is not None: + normalized = tuple(dict.fromkeys(self.track_ids)) + if not normalized: + raise ValueError("track_ids cannot be empty") + object.__setattr__(self, "track_ids", normalized) + object.__setattr__(self, "auto_select", False) + + +@dataclass(frozen=True) +class DownloadRequest: + """Output, selection, and download behavior for :meth:`N_m3u8DL.download`.""" + + output_dir: Path | str = field(default_factory=Path.cwd) + temporary_dir: Path | str | None = None + file_name: str | None = None + save_pattern: str | None = None + selection: TrackSelection = field(default_factory=TrackSelection) + thread_count: int = field(default_factory=lambda: max(1, os.cpu_count() or 1)) + retry_count: int = 3 + merge: bool = True + binary_merge: bool = False + delete_temporary_files: bool = True + check_segments_count: bool = True + max_speed: int | None = None + subtitle_format: str = "SRT" + auto_subtitle_fix: bool = True + custom_range: str | None = None + ad_keywords: tuple[str, ...] = () + ffmpeg_path: str | None = None + decryption_engine: str = "MP4DECRYPT" + decryption_binary_path: str | None = None + keys: tuple[str, ...] = () + + def __post_init__(self) -> None: + if self.thread_count < 1: + raise ValueError("thread_count must be at least one") + if self.retry_count < 0: + raise ValueError("retry_count cannot be negative") + if self.max_speed is not None and self.max_speed <= 0: + raise ValueError("max_speed must be greater than zero") + format_name = self.subtitle_format.upper() + if format_name not in {"SRT", "VTT"}: + raise ValueError("subtitle_format must be SRT or VTT") + engine = self.decryption_engine.upper() + if engine not in {"FFMPEG", "MP4DECRYPT", "SHAKA_PACKAGER"}: + raise ValueError("unsupported decryption_engine") + object.__setattr__(self, "output_dir", Path(self.output_dir)) + object.__setattr__(self, "temporary_dir", Path(self.temporary_dir) if self.temporary_dir is not None else None) + object.__setattr__(self, "subtitle_format", format_name) + object.__setattr__(self, "decryption_engine", engine) + object.__setattr__(self, "ad_keywords", tuple(self.ad_keywords)) + object.__setattr__(self, "keys", tuple(self.keys)) + + +@dataclass(frozen=True) +class MuxRequest: + """Optional ffmpeg mux configuration for completed download files.""" + + format: str = "mkv" + output_path: Path | str | None = None + ffmpeg_path: str | None = None + keep_inputs: bool = False + + def __post_init__(self) -> None: + format_name = self.format.lower() + if format_name not in {"mkv", "mp4", "ts"}: + raise ValueError("format must be mkv, mp4, or ts") + object.__setattr__(self, "format", format_name) + object.__setattr__(self, "output_path", Path(self.output_path) if self.output_path is not None else None) + + +@dataclass(frozen=True) +class MediaTrack: + """Immutable public description of one downloadable media track.""" + + track_id: str + media_type: MediaType + group_id: str | None + language: str | None + name: str | None + bandwidth: int | None + codecs: str | None + resolution: str | None + frame_rate: float | None + channels: str | None + extension: str | None + role: str | None + period_id: str | None + segments_count: int + duration: float + is_live: bool + is_encrypted: bool + + +@dataclass(frozen=True) +class MediaInfo: + """Result of manifest inspection. Pass this object to :meth:`N_m3u8DL.download`.""" + + input_value: str + protocol: str + source_url: str + tracks: tuple[MediaTrack, ...] + _source: ExtractedSource = field(repr=False, compare=False) + _streams_by_id: Mapping[str, StreamSpec] = field(repr=False, compare=False) + _request_options: RequestOptions = field(repr=False, compare=False) + + +@dataclass(frozen=True) +class DownloadedFile: + """One output generated by a successful download.""" + + track_id: str + media_type: MediaType + path: Path + size: int + + +@dataclass(frozen=True) +class DownloadResult: + """Successful download result. Failures are raised as exceptions instead.""" + + media_info: MediaInfo + selected_track_ids: tuple[str, ...] + files: tuple[DownloadedFile, ...] + temporary_dir: Path + + +@dataclass(frozen=True) +class MuxResult: + """Output generated by :meth:`N_m3u8DL.mux`.""" + + path: Path + inputs: tuple[DownloadedFile, ...] + + +class N_m3u8DL: + """Programmatic facade for manifest inspection, downloading, and muxing.""" + + def __init__(self, request_options: RequestOptions | None = None) -> None: + self.request_options = request_options or RequestOptions() + + def inspect(self, input_value: str | Path, *, options: RequestOptions | None = None, on_event: EventCallback | None = None) -> MediaInfo: + value = str(input_value) + request_options = options or self.request_options + try: + self._emit(on_event, DownloadEventKind.INSPECT_STARTED, message=f"Inspecting {value}") + client = HttpClient( + dict(request_options.headers), + request_options.timeout, + request_options.proxy, + request_options.use_system_proxy, + ) + source = load_source( + value, + client, + base_url=request_options.base_url, + append_url_params=request_options.append_url_params, + custom_hls_method=request_options.custom_hls_method, + custom_hls_key=request_options.custom_hls_key, + custom_hls_iv=request_options.custom_hls_iv, + allow_hls_multi_ext_map=request_options.allow_hls_multi_ext_map, + ) + except DownloadCancelledError: + self._emit_cancelled(on_event) + raise + except Exception as error: + self._emit(on_event, DownloadEventKind.FAILED, message=str(error)) + raise ManifestError(f"Unable to inspect {value}: {error}") from error + streams_by_id = {self._track_id(source.kind, index): stream for index, stream in enumerate(source.streams)} + info = MediaInfo( + input_value=value, + protocol=source.kind, + source_url=source.source_url, + tracks=tuple(self._to_track(track_id, stream) for track_id, stream in streams_by_id.items()), + _source=source, + _streams_by_id=MappingProxyType(streams_by_id), + _request_options=request_options, + ) + self._emit(on_event, DownloadEventKind.INSPECT_COMPLETED, message=f"Found {len(info.tracks)} tracks") + return info + + def download_url( + self, + input_value: str | Path, + request: DownloadRequest | None = None, + *, + options: RequestOptions | None = None, + on_event: EventCallback | None = None, + ) -> DownloadResult: + """Inspect and download one manifest without exposing the intermediate ``MediaInfo``.""" + media_info = self.inspect(input_value, options=options, on_event=on_event) + return self.download(media_info, request, on_event=on_event) + + def download(self, media_info: MediaInfo, request: DownloadRequest | None = None, *, on_event: EventCallback | None = None) -> DownloadResult: + request = request or DownloadRequest() + try: + self._emit(on_event, DownloadEventKind.DOWNLOAD_STARTED, message=f"Preparing {media_info.protocol} download") + selected_ids, selected_streams = self._select_streams(media_info, request.selection) + media_info._source.fetch_playlists(selected_streams) + self._emit(on_event, DownloadEventKind.PLAYLIST_LOADED, message=f"Loaded {len(selected_streams)} selected playlists") + apply_custom_range(selected_streams, request.custom_range) + clean_ads(selected_streams, list(request.ad_keywords)) + if not all(stream.playlist and stream.playlist.segments for stream in selected_streams): + raise SelectionError("One or more selected tracks have no downloadable media segments") + output_dir = request.output_dir + save_name = valid_filename(request.file_name or inferred_name(media_info.input_value), 180) + temporary_root = (request.temporary_dir or output_dir / ".n_m3u8dl") / save_name + ffmpeg = request.ffmpeg_path or shutil.which("ffmpeg") + decryption_binary = request.decryption_binary_path or self._find_decryption_binary(request.decryption_engine, ffmpeg) + options = DownloadOptions( + tmp_dir=temporary_root, + save_dir=output_dir, + save_name=save_name, + save_pattern=request.save_pattern, + thread_count=request.thread_count, + retries=request.retry_count, + binary_merge=request.binary_merge, + skip_merge=not request.merge, + del_after_done=request.delete_temporary_files, + check_segments_count=request.check_segments_count, + max_speed=request.max_speed, + subtitle_format=request.subtitle_format, + auto_subtitle_fix=request.auto_subtitle_fix, + ffmpeg=ffmpeg, + use_ffmpeg_concat_demuxer=True, + decryption_engine=request.decryption_engine, + decryption_binary=decryption_binary, + keys=list(request.keys), + ) + track_ids = {id(stream): track_id for track_id, stream in media_info._streams_by_id.items()} + manager = DownloadManager( + HttpClient( + dict(media_info._request_options.headers), + media_info._request_options.timeout, + media_info._request_options.proxy, + media_info._request_options.use_system_proxy, + ), + options, + event_callback=on_event, + track_ids=track_ids, + ) + outputs = manager.download(selected_streams, concurrent_tracks=False) + files = tuple( + DownloadedFile( + track_id=track_id, + media_type=self._media_type(stream), + path=path, + size=path.stat().st_size if path.is_file() else 0, + ) + for track_id, stream, path in zip(selected_ids, selected_streams, outputs, strict=True) + ) + except DownloadCancelledError: + self._emit_cancelled(on_event) + raise + except N_m3u8DLError as error: + self._emit(on_event, DownloadEventKind.FAILED, message=str(error)) + raise + except Exception as error: + self._emit(on_event, DownloadEventKind.FAILED, message=str(error)) + raise DownloadError(str(error)) from error + result = DownloadResult(media_info, tuple(selected_ids), files, temporary_root) + self._emit(on_event, DownloadEventKind.DOWNLOAD_COMPLETED, message=f"Created {len(files)} output file(s)") + return result + + def mux(self, result: DownloadResult, request: MuxRequest | None = None) -> MuxResult: + request = request or MuxRequest() + ffmpeg = request.ffmpeg_path or shutil.which("ffmpeg") + if not ffmpeg: + raise DownloadError("ffmpeg is required for muxing") + inputs = [item.path for item in result.files if item.path.is_file()] + if not inputs: + raise DownloadError("No downloaded files are available for muxing") + output_stem = request.output_path or result.files[0].path.with_name(valid_filename(inferred_name(result.media_info.input_value), 180)) + try: + output = mux_outputs(ffmpeg, inputs, output_stem, request.format, request.keep_inputs) + except Exception as error: + raise DownloadError(f"Mux failed: {error}") from error + return MuxResult(output, result.files) + + @staticmethod + def _track_id(protocol: str, index: int) -> str: + return f"{protocol.lower()}:{index}" + + @staticmethod + def _media_type(stream: StreamSpec) -> MediaType: + return stream.media_type or MediaType.VIDEO + + def _to_track(self, track_id: str, stream: StreamSpec) -> MediaTrack: + playlist = stream.playlist + segments = playlist.segments if playlist else [] + return MediaTrack( + track_id=track_id, + media_type=self._media_type(stream), + group_id=stream.group_id, + language=stream.language, + name=stream.name, + bandwidth=stream.bandwidth, + codecs=stream.codecs, + resolution=stream.resolution, + frame_rate=stream.frame_rate, + channels=stream.channels, + extension=stream.extension, + role=stream.role, + period_id=stream.period_id, + segments_count=len(segments), + duration=playlist.total_duration if playlist else 0.0, + is_live=playlist.is_live if playlist else False, + is_encrypted=any(segment.is_encrypted for segment in segments), + ) + + def _select_streams(self, media_info: MediaInfo, selection: TrackSelection) -> tuple[list[str], list[StreamSpec]]: + if selection.track_ids is not None: + unknown = [track_id for track_id in selection.track_ids if track_id not in media_info._streams_by_id] + if unknown: + raise SelectionError(f"Unknown track ID(s): {', '.join(unknown)}") + return list(selection.track_ids), [media_info._streams_by_id[track_id] for track_id in selection.track_ids] + if not selection.auto_select: + raise SelectionError("Specify track_ids or enable auto_select") + selected = automatic_selection(list(media_info._streams_by_id.values()), selection.sub_only) + if not selected: + raise SelectionError("Automatic selection did not find any matching tracks") + ids_by_stream = {id(stream): track_id for track_id, stream in media_info._streams_by_id.items()} + return [ids_by_stream[id(stream)] for stream in selected], selected + + @staticmethod + def _find_decryption_binary(engine: str, ffmpeg: str | None) -> str | None: + if engine == "FFMPEG": + return ffmpeg + if engine == "SHAKA_PACKAGER": + return shutil.which("shaka-packager") + return shutil.which("mp4decrypt") + + @staticmethod + def _emit(on_event: EventCallback | None, kind: DownloadEventKind, **kwargs: object) -> None: + if on_event is not None and on_event(DownloadEvent(kind, **kwargs)) is False: + raise DownloadCancelledError("Download cancelled by event callback") + + @staticmethod + def _emit_cancelled(on_event: EventCallback | None) -> None: + if on_event is not None: + on_event(DownloadEvent(DownloadEventKind.CANCELLED, message="Download cancelled")) diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/cli.py b/N_m3u8DL_py/src/n_m3u8dl_py/cli.py new file mode 100644 index 0000000..d885776 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/cli.py @@ -0,0 +1,334 @@ +from __future__ import annotations + +import argparse +import json +import shutil +import sys +import time +from datetime import datetime +from pathlib import Path + +from . import __version__ +from .downloader import DownloadManager, DownloadOptions, mux_outputs +from .extractor import ExtractedSource, load_source +from .http import HttpClient +from .models import EncryptMethod, MediaPart, MediaType, StreamSpec +from .selection import StreamFilter, apply_custom_range, apply_drop, automatic_selection, clean_ads +from .utils import inferred_name, parse_data, to_jsonable, valid_filename + + +VERSION_INFO = f"N_m3u8DL-PY (Python reimplementation) {__version__}" + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(prog="N_m3u8DL-PY", description="DASH/HLS/MSS download tool") + parser.add_argument("input", nargs="?", help="Input URL or manifest file") + general = parser.add_argument_group("general") + general.add_argument("--tmp-dir") + general.add_argument("--save-dir") + general.add_argument("--save-name") + general.add_argument("--save-pattern") + general.add_argument("--log-file-path") + general.add_argument("--base-url") + general.add_argument("--urlprocessor-args") + general.add_argument("--thread-count", type=int, default=max(1, __import__("os").cpu_count() or 1)) + general.add_argument("--download-retry-count", type=int, default=3) + general.add_argument("--http-request-timeout", type=float, default=100) + general.add_argument("-H", "--header", action="append", default=[], help='Custom HTTP header, e.g. -H "Cookie: value"') + general.add_argument("--custom-proxy") + general.add_argument("--use-system-proxy", action=argparse.BooleanOptionalAction, default=True) + general.add_argument("--append-url-params", action="store_true") + general.add_argument("--write-meta-json", action=argparse.BooleanOptionalAction, default=True) + general.add_argument("--no-log", action="store_true") + general.add_argument("--log-level", choices=["DEBUG", "INFO", "WARN", "ERROR", "OFF"], default="INFO") + general.add_argument("--ui-language", choices=["en-US", "zh-CN", "zh-TW"]) + general.add_argument("--force-ansi-console", action="store_true") + general.add_argument("--no-ansi-color", action="store_true") + general.add_argument("--disable-update-check", action="store_true") + general.add_argument("--task-start-at", help="yyyyMMddHHmmss") + download = parser.add_argument_group("download") + download.add_argument("--auto-select", action="store_true") + download.add_argument("--sub-only", action="store_true") + download.add_argument("-mt", "--concurrent-download", action="store_true") + download.add_argument("--skip-download", action="store_true") + download.add_argument("--skip-merge", action="store_true") + download.add_argument("--no-date-info", action="store_true") + download.add_argument("--binary-merge", action="store_true") + download.add_argument("--use-ffmpeg-concat-demuxer", action="store_true") + download.add_argument("--del-after-done", action=argparse.BooleanOptionalAction, default=True) + download.add_argument("--check-segments-count", action=argparse.BooleanOptionalAction, default=True) + download.add_argument("-R", "--max-speed", type=_parse_speed) + download.add_argument("--custom-range") + download.add_argument("--ad-keyword", action="append", default=[]) + media = parser.add_argument_group("media") + media.add_argument("--sub-format", choices=["SRT", "VTT"], default="SRT") + media.add_argument("--auto-subtitle-fix", action=argparse.BooleanOptionalAction, default=True) + media.add_argument("--ffmpeg-binary-path") + media.add_argument("--key", action="append", default=[]) + media.add_argument("--key-text-file") + media.add_argument("--decryption-engine", choices=["FFMPEG", "MP4DECRYPT", "SHAKA_PACKAGER"], default="MP4DECRYPT") + media.add_argument("--decryption-binary-path") + media.add_argument("--mp4-real-time-decryption", action="store_true") + media.add_argument("--custom-hls-method", choices=[item.value for item in EncryptMethod]) + media.add_argument("--custom-hls-key") + media.add_argument("--custom-hls-iv") + media.add_argument("--allow-hls-multi-ext-map", action="store_true") + media.add_argument("-M", "--mux-after-done", metavar="OPTIONS") + media.add_argument("--mux-import", action="append", default=[]) + filters = parser.add_argument_group("track selection") + filters.add_argument("-sv", "--select-video") + filters.add_argument("-sa", "--select-audio") + filters.add_argument("-ss", "--select-subtitle") + filters.add_argument("-dv", "--drop-video") + filters.add_argument("-da", "--drop-audio") + filters.add_argument("-ds", "--drop-subtitle") + live = parser.add_argument_group("live") + live.add_argument("--live-perform-as-vod", action="store_true") + live.add_argument("--live-record-limit") + live.add_argument("--live-wait-time", type=float) + live.add_argument("--live-take-count", type=int, default=16) + live.add_argument("--live-real-time-merge", action="store_true") + live.add_argument("--live-keep-segments", action=argparse.BooleanOptionalAction, default=True) + live.add_argument("--live-pipe-mux", action="store_true") + live.add_argument("--live-fix-vtt-by-audio", action="store_true") + parser.add_argument("--morehelp", metavar="OPTION") + parser.add_argument("--version", action="version", version=VERSION_INFO) + return parser + + +def main(argv: list[str] | None = None) -> int: + parser = build_parser() + args = parser.parse_args(argv) + if args.morehelp: + print(_more_help(args.morehelp)) + return 0 + if not args.input: + parser.print_help() + return 2 + try: + return _run(args) + except KeyboardInterrupt: + print("Interrupted.", file=sys.stderr) + return 130 + except Exception as error: + print(f"Failed: {error}", file=sys.stderr) + if args.log_level == "DEBUG": + raise + return 1 + + +def _run(args: argparse.Namespace) -> int: + if args.task_start_at: + start_at = datetime.strptime(args.task_start_at, "%Y%m%d%H%M%S") + while datetime.now() < start_at: + time.sleep(1) + headers = _headers(args.header) + headers.setdefault("User-Agent", "Mozilla/5.0 (N_m3u8DL-PY)") + if args.urlprocessor_args: + print("Warning: --urlprocessor-args was accepted, but no site-specific URL processors are bundled in the Python version.") + if args.mp4_real_time_decryption: + print("Warning: --mp4-real-time-decryption is not available; CENC decryption is attempted after merge.") + if args.live_real_time_merge or args.live_pipe_mux: + print("Warning: real-time live merging is not available; completed live recordings are merged after capture.") + if args.live_fix_vtt_by_audio: + print("Warning: --live-fix-vtt-by-audio is not available in the Python version.") + client = HttpClient(headers, args.http_request_timeout, args.custom_proxy, args.use_system_proxy) + source = load_source( + args.input, + client, + base_url=args.base_url, + append_url_params=args.append_url_params, + custom_hls_method=EncryptMethod(args.custom_hls_method) if args.custom_hls_method else None, + custom_hls_key=parse_data(args.custom_hls_key), + custom_hls_iv=parse_data(args.custom_hls_iv), + allow_hls_multi_ext_map=args.allow_hls_multi_ext_map, + ) + save_name = valid_filename(args.save_name or inferred_name(args.input), 180) + tmp_root = Path(args.tmp_dir or Path.cwd()) / save_name + if args.write_meta_json: + tmp_root.mkdir(parents=True, exist_ok=True) + (tmp_root / f"raw.{source.kind.lower()}").write_text(source.content, encoding="utf-8") + _write_json(tmp_root / "meta.json", source.streams) + print(f"Matched {source.kind}; found {len(source.streams)} stream(s).") + for number, stream in enumerate(source.streams, 1): + print(f"[{number}] {stream.describe()}") + selected = _select(source, args) + if not selected: + raise ValueError("No streams selected") + source.fetch_playlists(selected) + if any(stream.playlist and stream.playlist.is_live for stream in selected) and not args.live_perform_as_vod: + _collect_live_snapshot(source, selected, args) + apply_custom_range(selected, args.custom_range) + clean_ads(selected, args.ad_keyword) + if args.write_meta_json: + _write_json(tmp_root / "meta_selected.json", selected) + print("Selected streams:") + for stream in selected: + print(f" {stream.describe()}") + if args.skip_download: + return 0 + ffmpeg = args.ffmpeg_binary_path or shutil.which("ffmpeg") + keys = _keys(args.key, args.key_text_file) + decryption_binary = args.decryption_binary_path or (shutil.which("mp4decrypt") if args.decryption_engine == "MP4DECRYPT" else ffmpeg if args.decryption_engine == "FFMPEG" else shutil.which("shaka-packager")) + options = DownloadOptions( + tmp_dir=tmp_root, + save_dir=Path(args.save_dir or Path.cwd()), + save_name=save_name, + save_pattern=args.save_pattern, + thread_count=max(1, args.thread_count), + retries=max(0, args.download_retry_count), + binary_merge=args.binary_merge, + skip_merge=args.skip_merge, + del_after_done=args.del_after_done, + check_segments_count=args.check_segments_count, + max_speed=args.max_speed, + subtitle_format=args.sub_format, + auto_subtitle_fix=args.auto_subtitle_fix, + ffmpeg=ffmpeg, + use_ffmpeg_concat_demuxer=args.use_ffmpeg_concat_demuxer, + decryption_engine=args.decryption_engine, + decryption_binary=decryption_binary, + keys=keys, + ) + manager = DownloadManager(client, options, print) + outputs = manager.download(selected, args.concurrent_download) + if args.mux_after_done: + if not ffmpeg: + raise FileNotFoundError("ffmpeg is required for --mux-after-done") + mux = _option_values(args.mux_after_done) + imported = [_option_values(value).get("path") for value in args.mux_import] + inputs = [item for item in outputs + [Path(item) for item in imported if item] if item.is_file()] + if not inputs: + raise ValueError("No completed files are available for muxing") + output = mux_outputs(ffmpeg, inputs, options.save_dir / save_name, mux.get("format", "mkv"), mux.get("keep", "false").lower() == "true") + print(f"Muxed {output}") + print("Done") + return 0 + + +def _select(source: ExtractedSource, args: argparse.Namespace) -> list[StreamSpec]: + streams = source.streams + videos = [stream for stream in streams if stream.media_type is None] + audios = [stream for stream in streams if stream.media_type is MediaType.AUDIO] + subtitles = [stream for stream in streams if stream.media_type is MediaType.SUBTITLES] + videos = apply_drop(videos, StreamFilter.parse(args.drop_video)) + audios = apply_drop(audios, StreamFilter.parse(args.drop_audio)) + subtitles = apply_drop(subtitles, StreamFilter.parse(args.drop_subtitle)) + has_keep_filter = any([args.select_video, args.select_audio, args.select_subtitle]) + if args.auto_select or args.sub_only: + return automatic_selection(videos + audios + subtitles, args.sub_only) + if has_keep_filter: + chosen: list[StreamSpec] = [] + for candidates, raw in ((videos, args.select_video), (audios, args.select_audio), (subtitles, args.select_subtitle)): + stream_filter = StreamFilter.parse(raw) + if stream_filter: + chosen.extend(stream_filter.select(candidates)) + return chosen + available = videos + audios + subtitles + if not sys.stdin.isatty() or len(available) == 1: + return automatic_selection(available) + return _interactive_selection(available) + + +def _interactive_selection(streams: list[StreamSpec]) -> list[StreamSpec]: + raw = input("Select comma-separated stream numbers (Enter = automatic): ").strip() + if not raw: + return automatic_selection(streams) + selections = {int(item.strip()) for item in raw.split(",") if item.strip()} + return [stream for index, stream in enumerate(streams, 1) if index in selections] + + +def _collect_live_snapshot(source: ExtractedSource, selected: list[StreamSpec], args: argparse.Namespace) -> None: + if source.kind != "HLS": + print("Live DASH/MSS refresh is not available; downloading the current manifest snapshot.") + return + if not args.live_record_limit: + print("Live playlist detected. Set --live-record-limit to finish automatically; downloading the current snapshot.") + return + from .utils import parse_duration + + limit = parse_duration(args.live_record_limit) + collected: dict[int, list] = {id(stream): list(stream.playlist.segments[-args.live_take_count :]) for stream in selected if stream.playlist} + seen = {id(stream): {(segment.url, segment.start_range, segment.expect_length) for segment in segments} for stream, segments in ((stream, collected[id(stream)]) for stream in selected if stream.playlist)} + started = time.monotonic() + while time.monotonic() - started < limit: + refresh = min((stream.playlist.refresh_interval_ms for stream in selected if stream.playlist), default=5000) / 1000 + time.sleep(args.live_wait_time if args.live_wait_time is not None else refresh) + source.fetch_playlists(selected) + for stream in selected: + if not stream.playlist: + continue + for segment in stream.playlist.segments: + key = (segment.url, segment.start_range, segment.expect_length) + if key not in seen[id(stream)]: + seen[id(stream)].add(key) + collected[id(stream)].append(segment) + print(f"Live recording: {int(time.monotonic() - started)}/{int(limit)} seconds") + for stream in selected: + if stream.playlist: + stream.playlist.media_parts = [MediaPart(collected[id(stream)])] + stream.playlist.is_live = False + + +def _headers(values: list[str]) -> dict[str, str]: + headers: dict[str, str] = {} + for value in values: + key, separator, contents = value.partition(":") + if not separator or not key.strip(): + raise ValueError(f"Invalid header: {value!r}; expected 'Name: value'") + headers[key.strip()] = contents.strip() + return headers + + +def _parse_speed(value: str) -> int: + text = value.strip().upper() + multiplier = 1024 * 1024 if text.endswith("M") else 1024 if text.endswith("K") else None + if not multiplier: + raise argparse.ArgumentTypeError("speed must have M or K suffix, for example 15M or 100K") + return int(float(text[:-1]) * multiplier) + + +def _keys(values: list[str], key_file: str | None) -> list[str]: + keys = list(values) + if key_file: + for line in Path(key_file).read_text(encoding="utf-8").splitlines(): + line = line.strip() + if line and not line.startswith("#"): + keys.append(line) + return keys + + +def _write_json(path: Path, value: object) -> None: + path.write_text(json.dumps(to_jsonable(value), ensure_ascii=False, indent=2), encoding="utf-8") + + +def _option_values(value: str) -> dict[str, str]: + result: dict[str, str] = {} + current: list[str] = [] + quote = "" + pieces: list[str] = [] + for char in value: + if char in "\"'": + quote = "" if quote == char else char if not quote else quote + if char == ":" and not quote: + pieces.append("".join(current)) + current = [] + else: + current.append(char) + pieces.append("".join(current)) + for piece in pieces: + key, separator, item = piece.partition("=") + if separator: + result[key.strip().lower()] = item.strip().strip("\"'") + return result + + +def _more_help(option: str) -> str: + help_text = { + "mux-after-done": "-M format=mkv|mp4|ts:keep=true|false\nMuxes completed tracks with ffmpeg.", + "custom-range": "--custom-range 0-10, 10-, -99, or 05:00-20:00", + "select-video": "-sv id=REGEX:res=REGEX:codecs=REGEX:for=best|worst|all", + "select-audio": "-sa lang=REGEX:for=best|worst|all", + "select-subtitle": "-ss lang=REGEX:for=all", + } + return help_text.get(option.lstrip("-"), f"No extra help for {option}") diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/dash.py b/N_m3u8DL_py/src/n_m3u8dl_py/dash.py new file mode 100644 index 0000000..cb0f4c4 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/dash.py @@ -0,0 +1,343 @@ +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET +from collections.abc import Iterable +from datetime import datetime, timezone + +from .models import EncryptInfo, EncryptMethod, MediaPart, MediaSegment, MediaType, Playlist, StreamSpec +from .utils import append_query, combine_url, parse_iso8601_duration, parse_range, replace_vars + + +def _local(element: ET.Element) -> str: + return element.tag.rsplit("}", 1)[-1] + + +def _children(element: ET.Element, name: str) -> list[ET.Element]: + return [child for child in element if _local(child) == name] + + +def _child(element: ET.Element, name: str) -> ET.Element | None: + return next(iter(_children(element, name)), None) + + +def _attr(element: ET.Element | None, name: str, default: str | None = None) -> str | None: + return element.attrib.get(name, default) if element is not None else default + + +def _find_attr(element: ET.Element, name: str) -> str | None: + for key, value in element.attrib.items(): + if key.rsplit("}", 1)[-1] == name: + return value + return None + + +class DASHExtractor: + def __init__(self, source_url: str, base_url: str | None = None, append_url_params: bool = False) -> None: + self.source_url = source_url + self.base_url = base_url or source_url + self.append_url_params = append_url_params + + def extract(self, content: str) -> list[StreamSpec]: + root = ET.fromstring(content.strip()) + if _local(root) != "MPD": + raise ValueError("Bad MPD: expected MPD root element") + is_live = root.attrib.get("type") == "dynamic" + mpd_duration = parse_iso8601_duration(root.attrib.get("mediaPresentationDuration")) + root_base = self._extend_base(self.base_url, root) + streams: list[StreamSpec] = [] + periods = _children(root, "Period") + for period_number, period in enumerate(periods): + period_base = self._extend_base(root_base, period) + period_id = period.attrib.get("id", str(period_number)) + period_duration = parse_iso8601_duration(period.attrib.get("duration")) or mpd_duration + for adaptation in _children(period, "AdaptationSet"): + adaptation_base = self._extend_base(period_base, adaptation) + representations = _children(adaptation, "Representation") + for representation in representations: + stream = self._make_stream( + root, + period, + adaptation, + representation, + self._extend_base(adaptation_base, representation), + period_id, + period_duration, + is_live, + ) + self._merge_period_stream(streams, stream, is_live) + audios = [stream for stream in streams if stream.media_type is MediaType.AUDIO] + subtitles = [stream for stream in streams if stream.media_type is MediaType.SUBTITLES] + for stream in streams: + if stream.media_type is None and stream.resolution: + stream.audio_id = max(audios, key=lambda item: item.bandwidth or 0).group_id if audios else None + stream.subtitle_id = max(subtitles, key=lambda item: item.bandwidth or 0).group_id if subtitles else None + if self.append_url_params: + for stream in streams: + if stream.playlist and stream.playlist.media_init: + stream.playlist.media_init.url = append_query(self.source_url, stream.playlist.media_init.url) + if stream.playlist: + for segment in stream.playlist.segments: + segment.url = append_query(self.source_url, segment.url) + return streams + + def _make_stream( + self, + root: ET.Element, + period: ET.Element, + adaptation: ET.Element, + representation: ET.Element, + segment_base_url: str, + period_id: str, + period_duration: float, + is_live: bool, + ) -> StreamSpec: + mime_type = _attr(representation, "mimeType") or _attr(adaptation, "mimeType") or _attr(representation, "contentType") or _attr(adaptation, "contentType") or "" + content_type = (_attr(representation, "contentType") or _attr(adaptation, "contentType") or mime_type.split("/", 1)[0]).lower() + media_type = {"audio": MediaType.AUDIO, "text": MediaType.SUBTITLES, "subtitle": MediaType.SUBTITLES}.get(content_type) + codecs = _attr(representation, "codecs") or _attr(adaptation, "codecs") + if codecs in {"stpp", "wvtt"}: + media_type = MediaType.SUBTITLES + role = self._role(representation) or self._role(adaptation) + if role and role.lower().replace("-", "") in {"subtitle", "forcedsubtitle"}: + media_type = MediaType.SUBTITLES + extension = mime_type.split("/", 1)[1] if "/" in mime_type else None + if media_type is MediaType.SUBTITLES and "ttml" in mime_type: + extension = "ttml" + frame_rate = self._frame_rate(representation) or self._frame_rate(adaptation) + width = _attr(representation, "width") or _attr(adaptation, "width") + height = _attr(representation, "height") or _attr(adaptation, "height") + stream = StreamSpec( + url=self.source_url, + media_type=media_type, + group_id=_attr(representation, "id"), + bandwidth=int(_attr(representation, "bandwidth", "0") or 0), + codecs=codecs, + language=self._language(_attr(representation, "lang") or _attr(adaptation, "lang")), + resolution=f"{width}x{height}" if width and height else None, + frame_rate=frame_rate, + role=role, + period_id=period_id, + original_url=self.source_url, + extension=extension, + playlist=Playlist(is_live=is_live), + ) + stream.playlist.media_parts.append(MediaPart()) + channel_config = _child(representation, "AudioChannelConfiguration") + if channel_config is None: + channel_config = _child(adaptation, "AudioChannelConfiguration") + stream.channels = _attr(channel_config, "value") + publish_time = _attr(root, "publishTime") + if publish_time: + try: + stream.publish_time = datetime.fromisoformat(publish_time.replace("Z", "+00:00")) + except ValueError: + pass + buffer_depth = parse_iso8601_duration(_attr(root, "timeShiftBufferDepth") or "PT1M") + if is_live: + stream.playlist.refresh_interval_ms = max(1000, buffer_depth * 500) + + self._parse_segment_base(adaptation, representation, segment_base_url, stream, period_duration) + self._parse_segment_list(adaptation, representation, segment_base_url, stream) + self._parse_segment_template(root, period, adaptation, representation, segment_base_url, stream, period_duration, is_live) + if not stream.playlist.segments: + stream.playlist.media_parts[0].media_segments.append(MediaSegment(0, segment_base_url, period_duration)) + self._dedupe(stream.playlist.media_parts[0].media_segments) + self._apply_protection(adaptation, representation, stream) + if media_type is MediaType.SUBTITLES and extension == "mp4": + stream.extension = "m4s" + elif media_type is not MediaType.SUBTITLES and (not stream.extension or stream.segments_count > 1): + stream.extension = "m4s" + return stream + + def _parse_segment_base(self, adaptation: ET.Element, representation: ET.Element, base: str, stream: StreamSpec, duration: float) -> None: + segment_base = _child(representation, "SegmentBase") + if segment_base is None: + segment_base = _child(adaptation, "SegmentBase") + if segment_base is None: + return + initialization = _child(segment_base, "Initialization") + if initialization is None: + return + source = _attr(initialization, "sourceURL") + if not source: + stream.playlist.media_parts[0].media_segments.append(MediaSegment(0, base, duration)) + return + start_range, length = self._range(_attr(initialization, "range")) + stream.playlist.media_init = MediaSegment(-1, combine_url(base, source), start_range=start_range, expect_length=length) + + def _parse_segment_list(self, adaptation: ET.Element, representation: ET.Element, base: str, stream: StreamSpec) -> None: + segment_list = _child(representation, "SegmentList") + if segment_list is None: + segment_list = _child(adaptation, "SegmentList") + if segment_list is None: + return + timescale = int(_attr(segment_list, "timescale", "1") or 1) + duration = int(_attr(segment_list, "duration", "0") or 0) + initialization = _child(segment_list, "Initialization") + if initialization is not None: + start, length = self._range(_attr(initialization, "range")) + stream.playlist.media_init = MediaSegment(-1, combine_url(base, _attr(initialization, "sourceURL", "") or ""), start_range=start, expect_length=length) + for index, item in enumerate(_children(segment_list, "SegmentURL")): + start, length = self._range(_attr(item, "mediaRange")) + stream.playlist.media_parts[0].media_segments.append( + MediaSegment(index, combine_url(base, _attr(item, "media", "") or ""), duration / timescale, start_range=start, expect_length=length) + ) + + def _parse_segment_template( + self, + root: ET.Element, + period: ET.Element, + adaptation: ET.Element, + representation: ET.Element, + base: str, + stream: StreamSpec, + period_duration: float, + is_live: bool, + ) -> None: + templates = [_child(node, "SegmentTemplate") for node in (root, period, adaptation, representation)] + templates = [template for template in templates if template is not None] + if not templates: + return + attributes: dict[str, str] = {} + timeline: ET.Element | None = None + for template in templates: + attributes.update(template.attrib) + template_timeline = _child(template, "SegmentTimeline") + if template_timeline is not None: + timeline = template_timeline + values: dict[str, object | None] = { + "RepresentationID": stream.group_id, + "Bandwidth": stream.bandwidth, + } + timescale = int(attributes.get("timescale", "1")) + start_number = int(attributes.get("startNumber", "1")) + initialization = attributes.get("initialization") + media = attributes.get("media") + if initialization: + stream.playlist.media_init = MediaSegment(-1, combine_url(base, replace_vars(initialization, values))) + if not media: + return + if timeline is not None: + segment_values = self._timeline_segments(timeline, media, values, start_number, timescale, period_duration) + else: + duration = int(attributes.get("duration", "0")) + if not duration: + return + count = int((period_duration * timescale + duration - 1) // duration) if period_duration else (1 if is_live else 0) + segment_values = [(start_number + offset, offset * duration, duration) for offset in range(count)] + for index, (number, timestamp, duration) in enumerate(segment_values): + values["Number"] = number + values["Time"] = timestamp + stream.playlist.media_parts[0].media_segments.append( + MediaSegment(index, combine_url(base, replace_vars(media, values)), duration / timescale, name_from_var=str(timestamp) if "$Time" in media else str(number) if "$Number" in media else None) + ) + + def _timeline_segments( + self, + timeline: ET.Element, + media: str, + values: dict[str, object | None], + start_number: int, + timescale: int, + period_duration: float, + ) -> list[tuple[int, int, int]]: + items = _children(timeline, "S") + output: list[tuple[int, int, int]] = [] + current = 0 + number = start_number + for item_index, item in enumerate(items): + current = int(_attr(item, "t")) if _attr(item, "t") is not None else current + duration = int(_attr(item, "d", "0") or 0) + repeat = int(_attr(item, "r", "0") or 0) + if duration <= 0: + continue + if repeat < 0: + next_time = next((_attr(next_item, "t") for next_item in items[item_index + 1 :] if _attr(next_item, "t") is not None), None) + until = int(next_time) if next_time is not None else int(period_duration * timescale) + repeat = max(0, (until - current) // duration - 1) + for _ in range(repeat + 1): + output.append((number, current, duration)) + number += 1 + current += duration + return output + + @staticmethod + def _range(value: str | None) -> tuple[int | None, int | None]: + if not value: + return None, None + return parse_range(value) + + @staticmethod + def _extend_base(base: str, element: ET.Element) -> str: + base_url = _child(element, "BaseURL") + return combine_url(base, base_url.text or "") if base_url is not None else base + + @staticmethod + def _frame_rate(element: ET.Element) -> float | None: + value = _attr(element, "frameRate") + if not value: + return None + try: + numerator, separator, denominator = value.partition("/") + return float(numerator) / float(denominator) if separator else float(numerator) + except ValueError: + return None + + @staticmethod + def _language(value: str | None) -> str | None: + return value if value and re.fullmatch(r"[\w-]+", value) else "und" if value else None + + @staticmethod + def _role(element: ET.Element) -> str | None: + role = _child(element, "Role") + return _attr(role, "value") + + @staticmethod + def _dedupe(segments: list[MediaSegment]) -> None: + unique: list[MediaSegment] = [] + seen: set[tuple[str, int | None, int | None]] = set() + for segment in segments: + key = (segment.url, segment.start_range, segment.expect_length) + if key not in seen: + seen.add(key) + unique.append(segment) + segments[:] = unique + + def _apply_protection(self, adaptation: ET.Element, representation: ET.Element, stream: StreamSpec) -> None: + protection = _children(adaptation, "ContentProtection") + _children(representation, "ContentProtection") + if not protection: + return + kid: str | None = None + for element in protection: + value = _find_attr(element, "default_KID") + if value: + kid = value.replace("-", "").lower() + break + encrypted = EncryptInfo(EncryptMethod.CENC, kid=kid) + if stream.playlist.media_init: + stream.playlist.media_init.encrypt_info = encrypted.copy() + for segment in stream.playlist.segments: + segment.encrypt_info = encrypted.copy() + + @staticmethod + def _merge_period_stream(streams: list[StreamSpec], candidate: StreamSpec, is_live: bool) -> None: + existing = next( + ( + stream + for stream in streams + if stream.period_id != candidate.period_id + and stream.group_id == candidate.group_id + and stream.resolution == candidate.resolution + and stream.media_type == candidate.media_type + ), + None, + ) + if existing is None or is_live: + streams.append(candidate) + return + if existing.playlist and candidate.playlist: + offset = existing.segments_count + for segment in candidate.playlist.segments: + segment.index += offset + existing.playlist.media_parts.extend(candidate.playlist.media_parts) diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/downloader.py b/N_m3u8DL_py/src/n_m3u8dl_py/downloader.py new file mode 100644 index 0000000..2e28ac2 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/downloader.py @@ -0,0 +1,376 @@ +from __future__ import annotations + +import concurrent.futures +import shutil +import subprocess +import threading +import time +from dataclasses import dataclass +from pathlib import Path +from typing import Callable + +from .http import HttpClient +from .errors import DownloadCancelledError +from .events import DownloadEvent, DownloadEventKind, EventCallback +from .models import EncryptMethod, MediaSegment, MediaType, StreamSpec +from .subtitles import merge_webvtt +from .utils import valid_filename + + +@dataclass +class DownloadOptions: + tmp_dir: Path + save_dir: Path + save_name: str + save_pattern: str | None + thread_count: int + retries: int + binary_merge: bool + skip_merge: bool + del_after_done: bool + check_segments_count: bool + max_speed: int | None + subtitle_format: str + auto_subtitle_fix: bool + ffmpeg: str | None + use_ffmpeg_concat_demuxer: bool + decryption_engine: str + decryption_binary: str | None + keys: list[str] + + +class DownloadManager: + def __init__( + self, + client: HttpClient, + options: DownloadOptions, + report: Callable[[str], None] | None = None, + event_callback: EventCallback | None = None, + track_ids: dict[int, str] | None = None, + ) -> None: + self.client = client + self.options = options + self.report = report + self.event_callback = event_callback + self.track_ids = track_ids or {} + self.cancelled = threading.Event() + self._speed_started = time.monotonic() + self._speed_bytes = 0 + + def download(self, streams: list[StreamSpec], concurrent_tracks: bool = False) -> list[Path]: + self.options.tmp_dir.mkdir(parents=True, exist_ok=True) + self.options.save_dir.mkdir(parents=True, exist_ok=True) + if concurrent_tracks and len(streams) > 1: + workers = min(len(streams), max(1, self.options.thread_count)) + with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as executor: + futures = [executor.submit(self.download_stream, stream, number) for number, stream in enumerate(streams, 1)] + return [future.result() for future in futures] + return [self.download_stream(stream, number) for number, stream in enumerate(streams, 1)] + + def download_stream(self, stream: StreamSpec, number: int) -> Path: + self._check_cancelled() + if not stream.playlist or not stream.playlist.segments: + raise ValueError(f"No media segments for stream: {stream.describe()}") + label = valid_filename(f"{number}_{stream.group_id or 'stream'}_{stream.bandwidth or 0}_{stream.language or 'und'}", 160) + work_dir = self.options.tmp_dir / label + work_dir.mkdir(parents=True, exist_ok=True) + segments = stream.playlist.segments + width = max(1, len(str(len(segments)))) + self._report(f"Downloading {stream.describe()}") + self._emit(DownloadEventKind.TRACK_STARTED, stream, f"Downloading {stream.describe()}", total_segments=len(segments)) + self._check_cancelled() + downloaded: list[Path] = [] + init_path: Path | None = None + if stream.playlist.media_init: + init_path = work_dir / "00000_init.bin" + self._download_segment(stream.playlist.media_init, init_path) + with concurrent.futures.ThreadPoolExecutor(max_workers=max(1, self.options.thread_count)) as executor: + future_map = { + executor.submit(self._download_segment, segment, work_dir / f"{index:0{width}d}.bin"): index + for index, segment in enumerate(segments, 1) + } + paths: dict[int, Path] = {} + try: + for future in concurrent.futures.as_completed(future_map): + self._check_cancelled() + index = future_map[future] + paths[index] = future.result() + progress = len(paths) + self._report(f" {stream.type_name}: {progress}/{len(segments)} segments") + self._emit( + DownloadEventKind.SEGMENT_COMPLETED, + stream, + f"Downloaded segment {progress}/{len(segments)}", + completed_segments=progress, + total_segments=len(segments), + ) + self._check_cancelled() + except BaseException: + for future in future_map: + future.cancel() + raise + downloaded = [paths[index] for index in range(1, len(segments) + 1)] + if self.options.check_segments_count and len(downloaded) != len(segments): + raise RuntimeError(f"Segment count mismatch for {stream.short_name()}") + if self.options.skip_merge: + self._emit( + DownloadEventKind.TRACK_COMPLETED, + stream, + f"Downloaded segments to {work_dir}", + completed_segments=len(segments), + total_segments=len(segments), + output_path=work_dir, + ) + return work_dir + output = self._output_path(stream, number) + if stream.media_type is MediaType.SUBTITLES and self.options.auto_subtitle_fix: + self._merge_subtitles(downloaded, output) + else: + files = ([init_path] if init_path else []) + downloaded + force_binary = init_path is not None or self.options.binary_merge + output = self._merge_media([item for item in files if item], output, force_binary, stream) + if any(segment.encrypt_info.method is EncryptMethod.CENC for segment in segments): + self._decrypt_cenc(output, stream) + self._report(f"Saved {output}") + self._emit( + DownloadEventKind.TRACK_COMPLETED, + stream, + f"Saved {output}", + completed_segments=len(segments), + total_segments=len(segments), + output_path=output, + ) + if self.options.del_after_done: + shutil.rmtree(work_dir, ignore_errors=True) + return output + + def _download_segment(self, segment: MediaSegment, destination: Path) -> Path: + self._check_cancelled() + if destination.exists() and destination.stat().st_size: + return destination + payload, _, content_length = self.client.get_bytes( + segment.url, + start=segment.start_range, + end=segment.stop_range, + retries=self.options.retries, + ) + if segment.expect_length is not None and len(payload) != segment.expect_length: + raise IOError(f"Unexpected range size from {segment.url}: expected {segment.expect_length}, got {len(payload)}") + payload = self._decrypt_hls(payload, segment) + self._check_cancelled() + self._limit_speed(len(payload)) + temporary = destination.with_suffix(".part") + temporary.write_bytes(payload) + temporary.replace(destination) + return destination + + def _report(self, message: str) -> None: + if self.report is not None: + self.report(message) + + def _emit( + self, + kind: DownloadEventKind, + stream: StreamSpec, + message: str, + *, + completed_segments: int | None = None, + total_segments: int | None = None, + output_path: Path | None = None, + ) -> None: + if self.event_callback is None: + return + event = DownloadEvent( + kind, + message=message, + track_id=self.track_ids.get(id(stream)), + completed_segments=completed_segments, + total_segments=total_segments, + output_path=output_path, + ) + if self.event_callback(event) is False: + self.cancelled.set() + + def _check_cancelled(self) -> None: + if self.cancelled.is_set(): + raise DownloadCancelledError("Download cancelled by event callback") + + @staticmethod + def _decrypt_hls(payload: bytes, segment: MediaSegment) -> bytes: + encryption = segment.encrypt_info + if encryption.method not in {EncryptMethod.AES_128, EncryptMethod.AES_128_ECB}: + return payload + if not encryption.key: + raise ValueError(f"Missing key for encrypted segment {segment.url}") + openssl = shutil.which("openssl") + if not openssl: + raise RuntimeError("AES-128 HLS decryption requires the system 'openssl' command") + cipher_name = "aes-128-ecb" if encryption.method is EncryptMethod.AES_128_ECB else "aes-128-cbc" + command = [openssl, "enc", "-d", f"-{cipher_name}", "-K", encryption.key.hex(), "-nopad"] + if encryption.method is EncryptMethod.AES_128: + command.extend(["-iv", (encryption.iv or bytes(16)).hex()]) + result = subprocess.run(command, input=payload, capture_output=True) + if result.returncode: + raise RuntimeError(f"openssl AES decryption failed: {result.stderr.decode(errors='replace').strip()}") + plaintext = result.stdout + if plaintext: + padding = plaintext[-1] + if 0 < padding <= 16 and plaintext[-padding:] == bytes([padding]) * padding: + return plaintext[:-padding] + return plaintext + + def _limit_speed(self, byte_count: int) -> None: + if not self.options.max_speed: + return + self._speed_bytes += byte_count + target_elapsed = self._speed_bytes / self.options.max_speed + wait = target_elapsed - (time.monotonic() - self._speed_started) + if wait > 0: + time.sleep(wait) + + def _output_path(self, stream: StreamSpec, number: int) -> Path: + suffix = self._suffix(stream) + name = self._format_save_name(stream, number, suffix) + if not self.options.save_pattern and stream.language: + name += f".{valid_filename(stream.language, 30)}" + if not self.options.save_pattern and number > 1 and not stream.language: + name += f".{stream.type_name.lower()}" + filename = valid_filename(name, 180) + candidate = self.options.save_dir / (filename if filename.lower().endswith(suffix.lower()) else f"{filename}{suffix}") + if not candidate.exists(): + return candidate + sequence = 1 + while True: + alternate = candidate.with_name(f"{candidate.stem}_{sequence}{candidate.suffix}") + if not alternate.exists(): + return alternate + sequence += 1 + + def _format_save_name(self, stream: StreamSpec, number: int, suffix: str) -> str: + if not self.options.save_pattern: + return self.options.save_name + values = { + "SaveName": self.options.save_name, + "Id": stream.group_id or str(number), + "Codecs": stream.codecs or "", + "Language": stream.language or "und", + "Resolution": stream.resolution or "", + "Bandwidth": str(stream.bandwidth or ""), + "MediaType": stream.type_name.lower(), + "Channels": stream.channels or "", + "FrameRate": str(stream.frame_rate or ""), + "VideoRange": stream.video_range or "", + "GroupId": stream.group_id or "", + "Ext": suffix.lstrip("."), + } + result = self.options.save_pattern + for key, value in values.items(): + result = result.replace(f"<{key}>", value) + return result + + def _suffix(self, stream: StreamSpec) -> str: + if stream.media_type is MediaType.SUBTITLES and self.options.auto_subtitle_fix: + return ".srt" if self.options.subtitle_format.upper() == "SRT" else ".vtt" + if stream.media_type is MediaType.AUDIO and stream.extension in {"m4s", "mp4"}: + return ".m4a" + if stream.extension in {"m4s", "mp4"} and stream.media_type is not MediaType.SUBTITLES: + return ".mp4" + return f".{stream.extension or 'ts'}" + + def _merge_subtitles(self, files: list[Path], output: Path) -> None: + try: + output.write_text(merge_webvtt([item.read_bytes() for item in files], output.suffix == ".srt"), encoding="utf-8") + except Exception: + self._merge_binary(files, output) + + def _merge_media(self, files: list[Path], output: Path, force_binary: bool, stream: StreamSpec) -> Path: + if force_binary: + self._merge_binary(files, output) + return output + ffmpeg = self.options.ffmpeg + if not ffmpeg: + self._merge_binary(files, output) + return output + + ffmpeg_output = self._ffmpeg_output_path(output, stream) + manifest = ffmpeg_output.with_suffix(".concat.txt") + manifest.write_text("".join(f"file '{item.as_posix().replace("'", "'\\\\''")}'\n" for item in files), encoding="utf-8") + command = [ffmpeg, "-hide_banner", "-loglevel", "warning", "-nostdin", "-f", "concat", "-safe", "0", "-i", str(manifest), "-c", "copy", "-y", str(ffmpeg_output)] + result = subprocess.run(command, capture_output=True, text=True) + manifest.unlink(missing_ok=True) + if result.returncode != 0: + ffmpeg_output.unlink(missing_ok=True) + self._report(f"ffmpeg concat failed, using binary merge: {result.stderr.strip().splitlines()[-1:]}") + self._merge_binary(files, output) + return output + return ffmpeg_output + + def _ffmpeg_output_path(self, output: Path, stream: StreamSpec) -> Path: + suffix = ".m4a" if stream.media_type is MediaType.AUDIO else ".mp4" + candidate = output.with_suffix(suffix) + if not candidate.exists(): + return candidate + sequence = 1 + while True: + alternate = candidate.with_name(f"{candidate.stem}_{sequence}{candidate.suffix}") + if not alternate.exists(): + return alternate + sequence += 1 + + @staticmethod + def _merge_binary(files: list[Path], output: Path) -> None: + with output.open("wb") as target: + for source in files: + with source.open("rb") as incoming: + shutil.copyfileobj(incoming, target, 1024 * 1024) + + def _decrypt_cenc(self, output: Path, stream: StreamSpec) -> None: + if not self.options.keys: + self._report(f"Warning: CENC detected for {output}; downloaded media remains encrypted (no --key supplied).") + return + binary = self.options.decryption_binary + engine = self.options.decryption_engine.upper() + if not binary: + self._report(f"Warning: CENC detected for {output}; no decryption binary configured.") + return + decrypted = output.with_name(f"{output.stem}_dec{output.suffix}") + if engine == "MP4DECRYPT": + command = [binary] + for key in self.options.keys: + command.extend(["--key", key]) + command.extend([str(output), str(decrypted)]) + elif engine == "SHAKA_PACKAGER": + command = [binary, f"input={output},stream=0,output={decrypted}"] + for key in self.options.keys: + command.append(f"--keys=key_id={key.split(':', 1)[0]},key={key.split(':', 1)[-1]}") + else: + command = [binary, "-hide_banner", "-loglevel", "warning", "-y", "-decryption_key", self.options.keys[0].split(":")[-1], "-i", str(output), "-c", "copy", str(decrypted)] + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode: + self._report(f"Warning: external CENC decryption failed: {result.stderr.strip()[-500:]}") + return + decrypted.replace(output) + + +def mux_outputs(ffmpeg: str, inputs: list[Path], output_stem: Path, format_name: str = "mkv", keep: bool = False) -> Path: + format_name = format_name.lower() + if format_name not in {"mkv", "mp4", "ts"}: + raise ValueError("--mux-after-done format must be mkv, mp4 or ts") + output = output_stem.with_suffix(f".{format_name}") + command = [ffmpeg, "-hide_banner", "-loglevel", "warning", "-nostdin", "-y"] + for item in inputs: + command.extend(["-i", str(item)]) + command.extend(["-map", "0"]) + for index in range(1, len(inputs)): + command.extend(["-map", str(index)]) + command.extend(["-c", "copy"]) + if format_name == "mp4" and any(item.suffix.lower() == ".srt" for item in inputs): + command.extend(["-c:s", "mov_text"]) + command.append(str(output)) + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode: + raise RuntimeError(f"ffmpeg mux failed: {result.stderr[-1000:]}") + if not keep: + for item in inputs: + item.unlink(missing_ok=True) + return output diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/errors.py b/N_m3u8DL_py/src/n_m3u8dl_py/errors.py new file mode 100644 index 0000000..1fd5026 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/errors.py @@ -0,0 +1,21 @@ +"""Exceptions raised by the supported Python API.""" + + +class N_m3u8DLError(Exception): + """Base class for errors raised by the N_m3u8DL-PY API.""" + + +class ManifestError(N_m3u8DLError): + """The input could not be loaded or parsed as a supported manifest.""" + + +class SelectionError(N_m3u8DLError): + """The requested media tracks do not exist or cannot be downloaded.""" + + +class DownloadError(N_m3u8DLError): + """A selected media track could not be downloaded or merged.""" + + +class DownloadCancelledError(N_m3u8DLError): + """The caller cancelled a download by returning ``False`` from ``on_event``.""" diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/events.py b/N_m3u8DL_py/src/n_m3u8dl_py/events.py new file mode 100644 index 0000000..769fd35 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/events.py @@ -0,0 +1,38 @@ +"""Progress events for programmatic downloads.""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from enum import Enum +from pathlib import Path +from typing import Callable + + +class DownloadEventKind(str, Enum): + INSPECT_STARTED = "inspect_started" + INSPECT_COMPLETED = "inspect_completed" + PLAYLIST_LOADED = "playlist_loaded" + DOWNLOAD_STARTED = "download_started" + TRACK_STARTED = "track_started" + SEGMENT_COMPLETED = "segment_completed" + TRACK_COMPLETED = "track_completed" + DOWNLOAD_COMPLETED = "download_completed" + FAILED = "failed" + CANCELLED = "cancelled" + + +@dataclass(frozen=True) +class DownloadEvent: + """A synchronous event emitted while inspecting or downloading media.""" + + kind: DownloadEventKind + message: str = "" + track_id: str | None = None + completed_segments: int | None = None + total_segments: int | None = None + output_path: Path | None = None + timestamp: datetime = field(default_factory=lambda: datetime.now(timezone.utc)) + + +EventCallback = Callable[[DownloadEvent], bool | None] diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/extractor.py b/N_m3u8DL_py/src/n_m3u8dl_py/extractor.py new file mode 100644 index 0000000..96728be --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/extractor.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +from .dash import DASHExtractor +from .hls import HLSExtractor +from .http import HttpClient +from .models import EncryptMethod, StreamSpec +from .mss import MSSExtractor + + +@dataclass +class ExtractedSource: + kind: str + source_url: str + content: str + streams: list[StreamSpec] + extractor: HLSExtractor | DASHExtractor | MSSExtractor + + def fetch_playlists(self, streams: list[StreamSpec]) -> None: + if isinstance(self.extractor, HLSExtractor): + self.extractor.fetch_playlists(streams) + + +def load_source( + value: str, + client: HttpClient, + *, + base_url: str | None = None, + append_url_params: bool = False, + custom_hls_method: EncryptMethod | None = None, + custom_hls_key: bytes | None = None, + custom_hls_iv: bytes | None = None, + allow_hls_multi_ext_map: bool = False, +) -> ExtractedSource: + if value.startswith(("http://", "https://", "file:")): + content, source_url = client.get_text(value) + else: + path = Path(value).expanduser().resolve() + content = path.read_text(encoding="utf-8-sig") + source_url = path.as_uri() + text = content.lstrip("\ufeff").strip() + if text.startswith("#EXTM3U"): + extractor = HLSExtractor( + source_url, + client, + base_url, + append_url_params, + custom_hls_method, + custom_hls_key, + custom_hls_iv, + allow_hls_multi_ext_map, + ) + return ExtractedSource("HLS", source_url, text, extractor.extract(text), extractor) + if "" in text: + extractor = DASHExtractor(source_url, base_url, append_url_params) + return ExtractedSource("DASH", source_url, text, extractor.extract(text), extractor) + if "" in text: + extractor = MSSExtractor(source_url, base_url, append_url_params) + return ExtractedSource("MSS", source_url, text, extractor.extract(text), extractor) + raise ValueError("Input is not a supported HLS, DASH, or Smooth Streaming manifest") diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/hls.py b/N_m3u8DL_py/src/n_m3u8dl_py/hls.py new file mode 100644 index 0000000..3335333 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/hls.py @@ -0,0 +1,228 @@ +from __future__ import annotations + +from datetime import datetime +from urllib.parse import urlparse + +from .http import HttpClient +from .models import EncryptInfo, EncryptMethod, MediaPart, MediaSegment, MediaType, Playlist, StreamSpec +from .utils import append_query, combine_url, get_attribute, parse_hls_byterange + + +class HLSExtractor: + def __init__( + self, + source_url: str, + client: HttpClient, + base_url: str | None = None, + append_url_params: bool = False, + custom_method: EncryptMethod | None = None, + custom_key: bytes | None = None, + custom_iv: bytes | None = None, + allow_multi_ext_map: bool = False, + ) -> None: + self.source_url = source_url + self.client = client + self.base_url = base_url or source_url + self.append_url_params = append_url_params + self.custom_method = custom_method + self.custom_key = custom_key + self.custom_iv = custom_iv + self.allow_multi_ext_map = allow_multi_ext_map + self.is_master = False + + def extract(self, content: str) -> list[StreamSpec]: + content = self._preprocess(content) + if "#EXT-X-STREAM-INF" in content: + self.is_master = True + return self._parse_master(content) + playlist = self._parse_playlist(content, self.source_url) + return [StreamSpec(url=self.source_url, playlist=playlist, extension="mp4" if playlist.media_init else "ts")] + + def fetch_playlists(self, streams: list[StreamSpec]) -> None: + for stream in streams: + content, resolved_url = self.client.get_text(stream.url) + playlist = self._parse_playlist(self._preprocess(content), resolved_url) + if stream.playlist and stream.playlist.media_init: + playlist.media_init = stream.playlist.media_init + stream.playlist = playlist + if stream.media_type is MediaType.SUBTITLES: + urls = [segment.url.lower() for segment in playlist.segments] + stream.extension = "ttml" if any(".ttml" in url for url in urls) else "vtt" if any(".vtt" in url or ".webvtt" in url for url in urls) else "vtt" + else: + stream.extension = "m4s" if playlist.media_init else "ts" + + def _preprocess(self, content: str) -> str: + content = content.lstrip("\ufeff").strip() + if not content.startswith("#EXTM3U"): + raise ValueError("Bad m3u8: missing #EXTM3U") + return content + + def _url(self, base: str, value: str) -> str: + target = combine_url(base, value) + return append_query(self.source_url, target) if self.append_url_params else target + + def _parse_master(self, content: str) -> list[StreamSpec]: + streams: list[StreamSpec] = [] + pending: StreamSpec | None = None + for raw_line in content.splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("#EXT-X-STREAM-INF"): + codecs = get_attribute(line, "CODECS") or None + audio_id = get_attribute(line, "AUDIO") or None + if codecs and audio_id: + codecs = codecs.split(",", 1)[0] + frame_rate = get_attribute(line, "FRAME-RATE") + pending = StreamSpec( + url="", + bandwidth=int(get_attribute(line, "AVERAGE-BANDWIDTH") or get_attribute(line, "BANDWIDTH") or 0), + codecs=codecs, + resolution=get_attribute(line, "RESOLUTION") or None, + frame_rate=float(frame_rate) if frame_rate else None, + audio_id=audio_id, + video_id=get_attribute(line, "VIDEO") or None, + subtitle_id=get_attribute(line, "SUBTITLES") or None, + video_range=get_attribute(line, "VIDEO-RANGE") or None, + original_url=self.source_url, + ) + elif line.startswith("#EXT-X-MEDIA"): + type_name = get_attribute(line, "TYPE").replace("-", "_").upper() + if type_name == "CLOSED_CAPTIONS": + continue + try: + media_type = MediaType(type_name) + except ValueError: + continue + uri = get_attribute(line, "URI") + if not uri: + continue + default_text = get_attribute(line, "DEFAULT").upper() + streams.append( + StreamSpec( + url=self._url(self.base_url, uri), + media_type=media_type, + group_id=get_attribute(line, "GROUP-ID") or None, + language=get_attribute(line, "LANGUAGE") or None, + name=get_attribute(line, "NAME") or None, + default=default_text == "YES", + channels=get_attribute(line, "CHANNELS") or None, + characteristics=(get_attribute(line, "CHARACTERISTICS").split(",")[-1].split(".")[-1] or None), + original_url=self.source_url, + ) + ) + elif not line.startswith("#") and pending is not None: + pending.url = self._url(self.base_url, line) + streams.append(pending) + pending = None + unique: dict[str, StreamSpec] = {} + for stream in streams: + unique.setdefault(stream.url, stream) + return list(unique.values()) + + def _parse_playlist(self, content: str, playlist_url: str) -> Playlist: + base = self.base_url if self.base_url != self.source_url else playlist_url + playlist = Playlist(url=playlist_url) + parts: list[MediaPart] = [] + segments: list[MediaSegment] = [] + current_encryption = EncryptInfo( + self.custom_method or EncryptMethod.NONE, + self.custom_key, + self.custom_iv, + ) + pending_duration: float | None = None + pending_range: tuple[int, int] | None = None + previous_range_end: int | None = None + sequence = 0 + endlist = False + next_date: datetime | None = None + skip_ad = False + + for raw_line in content.splitlines(): + line = raw_line.strip() + if not line: + continue + if line.startswith("#EXT-X-TARGETDURATION"): + playlist.target_duration = float(get_attribute(line)) + elif line.startswith("#EXT-X-MEDIA-SEQUENCE"): + sequence = int(get_attribute(line)) + elif line.startswith("#EXT-X-PLAYLIST-TYPE"): + endlist = get_attribute(line).upper() == "VOD" + elif line.startswith("#EXT-X-PROGRAM-DATE-TIME"): + raw_date = get_attribute(line).replace("Z", "+00:00") + try: + next_date = datetime.fromisoformat(raw_date) + except ValueError: + next_date = None + elif line.startswith("#EXT-X-BYTERANGE"): + pending_range = parse_hls_byterange(get_attribute(line), previous_range_end) + previous_range_end = pending_range[0] + pending_range[1] + elif line.startswith("#EXT-X-DISCONTINUITY"): + if segments: + parts.append(MediaPart(segments)) + segments = [] + elif line.startswith("#EXT-X-KEY"): + current_encryption = self._parse_key(line, playlist_url, sequence, current_encryption) + elif line.startswith("#EXT-X-MAP"): + if playlist.media_init is not None and not self.allow_multi_ext_map: + break + init_url = self._url(base, get_attribute(line, "URI")) + init_range: tuple[int, int] | None = None + if get_attribute(line, "BYTERANGE"): + init_range = parse_hls_byterange(get_attribute(line, "BYTERANGE"), None) + playlist.media_init = MediaSegment( + index=-1, + url=init_url, + start_range=init_range[0] if init_range else None, + expect_length=init_range[1] if init_range else None, + encrypt_info=current_encryption.copy(), + ) + elif line.startswith("#EXTINF"): + value = get_attribute(line) + duration, _, _title = value.partition(",") + pending_duration = float(duration) + elif line.startswith("#UPLYNK-SEGMENT"): + skip_ad = ",ad" in line + elif line.startswith("#EXT-X-ENDLIST"): + endlist = True + elif not line.startswith("#") and pending_duration is not None: + url = self._url(base, line) + if not skip_ad and not ("/ad/" in url and "duration=" in url): + segments.append( + MediaSegment( + index=sequence, + url=url, + duration=pending_duration, + date_time=next_date, + start_range=pending_range[0] if pending_range else None, + expect_length=pending_range[1] if pending_range else None, + encrypt_info=current_encryption.copy(), + ) + ) + sequence += 1 + pending_duration = None + pending_range = None + next_date = None + if segments: + parts.append(MediaPart(segments)) + playlist.media_parts = parts + playlist.is_live = not endlist + if playlist.is_live: + playlist.refresh_interval_ms = (playlist.target_duration or 5) * 2000 + return playlist + + def _parse_key(self, line: str, playlist_url: str, sequence: int, current: EncryptInfo) -> EncryptInfo: + method = EncryptMethod.from_manifest(get_attribute(line, "METHOD")) + if method is EncryptMethod.NONE: + return EncryptInfo(EncryptMethod.NONE) + if self.custom_method: + method = self.custom_method + key = self.custom_key + key_uri = get_attribute(line, "URI") + if key is None and key_uri and method in {EncryptMethod.AES_128, EncryptMethod.AES_128_ECB, EncryptMethod.CHACHA20}: + key, _, _ = self.client.get_bytes(self._url(playlist_url, key_uri), retries=2) + iv_value = get_attribute(line, "IV") + iv = self.custom_iv or (bytes.fromhex(iv_value.removeprefix("0x")) if iv_value else sequence.to_bytes(16, "big")) + if method in {EncryptMethod.AES_128, EncryptMethod.AES_128_ECB} and not key: + raise ValueError("HLS key URI did not yield a decryption key") + return EncryptInfo(method, key, iv, current.kid) diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/http.py b/N_m3u8DL_py/src/n_m3u8dl_py/http.py new file mode 100644 index 0000000..bc77e10 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/http.py @@ -0,0 +1,234 @@ +from __future__ import annotations + +import base64 +import gzip +import ipaddress +import io +import socket +import time +from http.client import HTTPConnection, HTTPSConnection +from pathlib import Path +from typing import Mapping +from urllib.error import HTTPError, URLError +from urllib.parse import unquote, urlparse +from urllib.request import HTTPHandler, HTTPSHandler, Request, ProxyHandler, build_opener, url2pathname + + +class HttpClient: + """Small synchronous HTTP client shared by extractors and download workers.""" + + def __init__( + self, + headers: Mapping[str, str] | None = None, + timeout: float = 100, + proxy: str | None = None, + use_system_proxy: bool = True, + ) -> None: + self.headers = dict(headers or {}) + self.timeout = timeout + self.proxy = proxy + self.use_system_proxy = use_system_proxy + self.opener = build_proxy_opener(proxy, use_system_proxy) + + def get_text(self, url: str) -> tuple[str, str]: + data, resolved, _ = self.get_bytes(url) + return data.decode("utf-8-sig", errors="replace"), resolved + + def get_bytes( + self, + url: str, + headers: Mapping[str, str] | None = None, + start: int | None = None, + end: int | None = None, + retries: int = 3, + ) -> tuple[bytes, str, int | None]: + if url.startswith("file:"): + path = Path(url2pathname(unquote(urlparse(url).path))) + data = path.read_bytes() + return self._slice(data, start, end), url, len(data) + if url.startswith("base64://"): + data = base64.b64decode(url[len("base64://") :]) + return self._slice(data, start, end), url, len(data) + if url.startswith("hex://"): + data = bytes.fromhex(url[len("hex://") :]) + return self._slice(data, start, end), url, len(data) + + request_headers = dict(self.headers) + request_headers.update(headers or {}) + if start is not None or end is not None: + request_headers["Range"] = f"bytes={start or 0}-{'' if end is None else end}" + last_error: Exception | None = None + for attempt in range(retries + 1): + try: + request = Request(url, headers=request_headers, method="GET") + with self.opener.open(request, timeout=self.timeout) as response: + data = response.read() + if response.headers.get("Content-Encoding", "").lower() == "gzip": + data = gzip.GzipFile(fileobj=io.BytesIO(data)).read() + length = response.headers.get("Content-Length") + return data, response.geturl(), int(length) if length else None + except (HTTPError, URLError, TimeoutError, OSError) as error: + last_error = error + if attempt < retries: + time.sleep(min(1.0 * (attempt + 1), 3.0)) + assert last_error is not None + raise last_error + + @staticmethod + def _slice(data: bytes, start: int | None, end: int | None) -> bytes: + if start is None and end is None: + return data + return data[start or 0 : None if end is None else end + 1] + + +def build_proxy_opener(proxy: str | None, use_system_proxy: bool = True): + """Build an opener for direct, HTTP(S), or no-auth SOCKS5 proxy requests. + + SOCKS target hostnames are resolved by the proxy so downloads do not depend + on the downloader host's DNS configuration. + """ + if not proxy: + return build_opener(ProxyHandler({}) if not use_system_proxy else ProxyHandler()) + parsed = urlparse(proxy) + if parsed.scheme in {"socks5", "socks5h"}: + if parsed.username is not None or parsed.password is not None: + raise ValueError("Authenticated SOCKS5 proxies are not supported") + if not parsed.hostname: + raise ValueError("SOCKS5 proxy host is required") + port = parsed.port or 1080 + return build_opener( + ProxyHandler({}), + _Socks5HTTPHandler(parsed.hostname, port), + _Socks5HTTPSHandler(parsed.hostname, port), + ) + if parsed.scheme not in {"http", "https"}: + raise ValueError("Unsupported proxy scheme") + return build_opener(ProxyHandler({"http": proxy, "https": proxy})) + + +class _Socks5HTTPConnection(HTTPConnection): + def __init__(self, host: str, port: int | None = None, *, proxy_host: str, proxy_port: int, **kwargs: object) -> None: + super().__init__(host, port, **kwargs) + self._proxy_host = proxy_host + self._proxy_port = proxy_port + + def connect(self) -> None: + self.sock = _socks5_connect( + self._proxy_host, + self._proxy_port, + self.host, + self.port, + self.timeout, + self.source_address, + ) + + +class _Socks5HTTPSConnection(HTTPSConnection): + def __init__(self, host: str, port: int | None = None, *, proxy_host: str, proxy_port: int, **kwargs: object) -> None: + super().__init__(host, port, **kwargs) + self._proxy_host = proxy_host + self._proxy_port = proxy_port + + def connect(self) -> None: + socket_connection = _socks5_connect( + self._proxy_host, + self._proxy_port, + self.host, + self.port, + self.timeout, + self.source_address, + ) + self.sock = self._context.wrap_socket(socket_connection, server_hostname=self.host) + + +class _Socks5HTTPHandler(HTTPHandler): + def __init__(self, proxy_host: str, proxy_port: int) -> None: + super().__init__() + self._proxy_host = proxy_host + self._proxy_port = proxy_port + + def http_open(self, request: Request): + return self.do_open( + lambda host, **kwargs: _Socks5HTTPConnection( + host, + proxy_host=self._proxy_host, + proxy_port=self._proxy_port, + **kwargs, + ), + request, + ) + + +class _Socks5HTTPSHandler(HTTPSHandler): + def __init__(self, proxy_host: str, proxy_port: int) -> None: + super().__init__() + self._proxy_host = proxy_host + self._proxy_port = proxy_port + + def https_open(self, request: Request): + return self.do_open( + lambda host, **kwargs: _Socks5HTTPSConnection( + host, + proxy_host=self._proxy_host, + proxy_port=self._proxy_port, + **kwargs, + ), + request, + context=self._context, + ) + + +def _socks5_connect( + proxy_host: str, + proxy_port: int, + host: str, + port: int, + timeout: float | object, + source_address: tuple[str, int] | None, +) -> socket.socket: + connection = socket.create_connection((proxy_host, proxy_port), timeout=timeout, source_address=source_address) + try: + connection.sendall(b"\x05\x01\x00") + if _receive_exact(connection, 2) != b"\x05\x00": + raise OSError("SOCKS5 proxy does not support no-authentication") + address_type, address = _socks5_target(host) + connection.sendall(b"\x05\x01\x00" + bytes([address_type]) + address + port.to_bytes(2, "big")) + reply = _receive_exact(connection, 4) + if reply[0] != 5 or reply[1] != 0: + raise OSError(f"SOCKS5 CONNECT failed with code {reply[1]}") + if reply[3] == 1: + _receive_exact(connection, 4) + elif reply[3] == 4: + _receive_exact(connection, 16) + elif reply[3] == 3: + _receive_exact(connection, _receive_exact(connection, 1)[0]) + else: + raise OSError("SOCKS5 proxy returned an invalid bind address") + _receive_exact(connection, 2) + return connection + except BaseException: + connection.close() + raise + + +def _socks5_target(host: str) -> tuple[int, bytes]: + try: + address = ipaddress.ip_address(host) + except ValueError: + encoded = host.encode("idna") + if not encoded or len(encoded) > 255: + raise ValueError("SOCKS5 target hostname is invalid") + return 3, bytes([len(encoded)]) + encoded + return (1 if address.version == 4 else 4), address.packed + + +def _receive_exact(connection: socket.socket, count: int) -> bytes: + chunks: list[bytes] = [] + remaining = count + while remaining: + chunk = connection.recv(remaining) + if not chunk: + raise OSError("SOCKS5 proxy closed the connection") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/models.py b/N_m3u8DL_py/src/n_m3u8dl_py/models.py new file mode 100644 index 0000000..b442014 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/models.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime +from enum import Enum +from typing import Any + + +class MediaType(str, Enum): + VIDEO = "VIDEO" + AUDIO = "AUDIO" + SUBTITLES = "SUBTITLES" + CLOSED_CAPTIONS = "CLOSED_CAPTIONS" + + +class EncryptMethod(str, Enum): + NONE = "NONE" + AES_128 = "AES_128" + AES_128_ECB = "AES_128_ECB" + CENC = "CENC" + CHACHA20 = "CHACHA20" + SAMPLE_AES = "SAMPLE_AES" + SAMPLE_AES_CTR = "SAMPLE_AES_CTR" + UNKNOWN = "UNKNOWN" + + @classmethod + def from_manifest(cls, value: str | None) -> "EncryptMethod": + if not value: + return cls.UNKNOWN + normalized = value.upper().replace("-", "_") + return cls._value2member_map_.get(normalized, cls.UNKNOWN) + + +@dataclass +class EncryptInfo: + method: EncryptMethod = EncryptMethod.NONE + key: bytes | None = None + iv: bytes | None = None + kid: str | None = None + + def copy(self) -> "EncryptInfo": + return EncryptInfo(self.method, self.key, self.iv, self.kid) + + +@dataclass +class MediaSegment: + index: int + url: str + duration: float = 0.0 + title: str | None = None + date_time: datetime | None = None + start_range: int | None = None + expect_length: int | None = None + encrypt_info: EncryptInfo = field(default_factory=EncryptInfo) + name_from_var: str | None = None + + @property + def stop_range(self) -> int | None: + if self.start_range is None or self.expect_length is None: + return None + return self.start_range + self.expect_length - 1 + + @property + def is_encrypted(self) -> bool: + return self.encrypt_info.method is not EncryptMethod.NONE + + +@dataclass +class MediaPart: + media_segments: list[MediaSegment] = field(default_factory=list) + + +@dataclass +class Playlist: + url: str = "" + is_live: bool = False + refresh_interval_ms: float = 15000 + target_duration: float | None = None + media_init: MediaSegment | None = None + media_parts: list[MediaPart] = field(default_factory=list) + + @property + def segments(self) -> list[MediaSegment]: + return [segment for part in self.media_parts for segment in part.media_segments] + + @property + def total_duration(self) -> float: + return sum(segment.duration for segment in self.segments) + + +@dataclass +class StreamSpec: + url: str + media_type: MediaType | None = None + group_id: str | None = None + language: str | None = None + name: str | None = None + default: bool | None = None + bandwidth: int | None = None + codecs: str | None = None + resolution: str | None = None + frame_rate: float | None = None + channels: str | None = None + extension: str | None = None + role: str | None = None + video_range: str | None = None + characteristics: str | None = None + publish_time: datetime | None = None + audio_id: str | None = None + video_id: str | None = None + subtitle_id: str | None = None + period_id: str | None = None + original_url: str = "" + playlist: Playlist | None = None + skipped_duration: float = 0.0 + extra: dict[str, Any] = field(default_factory=dict) + + @property + def segments_count(self) -> int: + return len(self.playlist.segments) if self.playlist else 0 + + @property + def type_name(self) -> str: + return self.media_type.value.title() if self.media_type else "VIDEO" + + def short_name(self) -> str: + if self.media_type is MediaType.AUDIO: + fields = ["Aud", self.group_id, _kbps(self.bandwidth), self.name, self.codecs, self.language, self.channels] + elif self.media_type is MediaType.SUBTITLES: + fields = ["Sub", self.group_id, self.language, self.name, self.codecs] + else: + fields = ["Vid", self.resolution, _kbps(self.bandwidth), self.group_id, self.frame_rate, self.codecs] + return " | ".join(str(item) for item in fields if item not in (None, "")) + + def describe(self) -> str: + duration = _format_duration(self.playlist.total_duration) if self.playlist else "0s" + segments = f"{self.segments_count} segments" if self.segments_count else "" + return " | ".join(part for part in [self.short_name(), segments, f"~{duration}"] if part) + + +def _kbps(value: int | None) -> str | None: + return f"{value // 1000} Kbps" if value is not None else None + + +def _format_duration(total: float) -> str: + seconds = max(0, int(total)) + hours, seconds = divmod(seconds, 3600) + minutes, seconds = divmod(seconds, 60) + if hours: + return f"{hours}h{minutes}m{seconds}s" + if minutes: + return f"{minutes}m{seconds}s" + return f"{seconds}s" diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/mss.py b/N_m3u8DL_py/src/n_m3u8dl_py/mss.py new file mode 100644 index 0000000..97dd303 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/mss.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import re +import xml.etree.ElementTree as ET + +from .dash import _attr, _child, _children, _local +from .models import EncryptInfo, EncryptMethod, MediaPart, MediaSegment, MediaType, Playlist, StreamSpec +from .utils import append_query, combine_url, replace_vars + + +class MSSExtractor: + """Parse Smooth Streaming manifests into the common playlist model. + + Unlike the .NET edition, this implementation does not synthesize a `moov` + init segment from CodecPrivateData. Existing fragment streams remain useful + with `--binary-merge`, and an external ffmpeg may handle site-provided init + metadata where available. + """ + + def __init__(self, source_url: str, base_url: str | None = None, append_url_params: bool = False) -> None: + self.source_url = source_url + self.base_url = base_url or source_url + self.append_url_params = append_url_params + + def extract(self, content: str) -> list[StreamSpec]: + root = ET.fromstring(content.strip()) + if _local(root) != "SmoothStreamingMedia": + raise ValueError("Bad Smooth Streaming manifest") + timescale = int(_attr(root, "TimeScale", "10000000") or 10000000) + duration = int(_attr(root, "Duration", "0") or 0) + is_live = (_attr(root, "IsLive", "FALSE") or "FALSE").upper() == "TRUE" + protected = _child(root, "Protection") is not None + streams: list[StreamSpec] = [] + for stream_index in _children(root, "StreamIndex"): + stream_type = (_attr(stream_index, "Type") or "video").lower() + media_type = {"audio": MediaType.AUDIO, "text": MediaType.SUBTITLES}.get(stream_type) + url_pattern = (_attr(stream_index, "Url") or "").replace("{bitrate}", "$Bandwidth$").replace("{Bitrate}", "$Bandwidth$").replace("{start time}", "$Time$").replace("{start_time}", "$Time$") + language = _attr(stream_index, "Language") + language = language if language and len(language) == 3 else None + chunks = _children(stream_index, "c") + for quality in _children(stream_index, "QualityLevel"): + bitrate = int(_attr(quality, "Bitrate", "0") or 0) + codec_private_data = _attr(quality, "CodecPrivateData", "") or "" + fourcc = (_attr(quality, "FourCC", "") or "").upper() + stream = StreamSpec( + url=self.source_url, + media_type=media_type, + group_id=_attr(stream_index, "Name") or _attr(quality, "Index"), + bandwidth=bitrate, + codecs=self._codecs(fourcc, codec_private_data), + language=language, + resolution=self._resolution(quality), + channels=_attr(quality, "Channels"), + period_id=_attr(quality, "Index"), + original_url=self.source_url, + extension="m4s", + playlist=Playlist(is_live=is_live, media_parts=[MediaPart()]), + extra={"mss_codec_private_data": codec_private_data, "mss_fourcc": fourcc}, + ) + current = 0 + index = 0 + for chunk_index, chunk in enumerate(chunks): + current = int(_attr(chunk, "t")) if _attr(chunk, "t") is not None else current + chunk_duration = int(_attr(chunk, "d", "0") or 0) + repeat = int(_attr(chunk, "r", "0") or 0) + if repeat > 0: + repeat -= 1 + if repeat < 0 and chunk_duration: + repeat = max(0, duration // chunk_duration - 1) + for _ in range(repeat + 1): + values = {"Bandwidth": bitrate, "Time": current} + url = combine_url(self.base_url, replace_vars(url_pattern, values)) + encryption = EncryptInfo(EncryptMethod.CENC) if protected and stream_type != "text" else EncryptInfo() + stream.playlist.media_parts[0].media_segments.append(MediaSegment(index, url, chunk_duration / timescale, encrypt_info=encryption, name_from_var=str(current))) + current += chunk_duration + index += 1 + streams.append(stream) + audios = [stream for stream in streams if stream.media_type is MediaType.AUDIO] + subtitles = [stream for stream in streams if stream.media_type is MediaType.SUBTITLES] + for stream in streams: + if stream.media_type is None: + stream.audio_id = audios[0].group_id if audios else None + stream.subtitle_id = subtitles[0].group_id if subtitles else None + if self.append_url_params: + for stream in streams: + for segment in stream.playlist.segments: + segment.url = append_query(self.source_url, segment.url) + return streams + + @staticmethod + def _resolution(quality: ET.Element) -> str | None: + width = _attr(quality, "MaxWidth") + height = _attr(quality, "MaxHeight") + return f"{width}x{height}" if width and height and width != "0" else None + + @staticmethod + def _codecs(fourcc: str, private_data: str) -> str | None: + if fourcc == "TTML": + return "stpp" + if fourcc in {"H264", "X264", "DAVC", "AVC1"}: + match = re.search(r"00000001d7([0-9a-fA-F]{6})", private_data) + return f"avc1.{match.group(1)}" if match else "avc1.4D401E" + if fourcc in {"AAC", "AACL", "AACH", "AACP"}: + profile = 5 if fourcc == "AACH" else int(private_data[:2], 16) >> 3 if private_data else 2 + return f"mp4a.40.{profile}" + return fourcc.lower() or None diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/selection.py b/N_m3u8DL_py/src/n_m3u8dl_py/selection.py new file mode 100644 index 0000000..3f2120b --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/selection.py @@ -0,0 +1,163 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass + +from .models import MediaSegment, MediaType, StreamSpec +from .utils import parse_duration + + +@dataclass +class StreamFilter: + values: dict[str, str] + selection: str = "best" + + @classmethod + def parse(cls, raw: str | None) -> "StreamFilter | None": + if raw is None: + return None + if raw in {"all", "best", "worst"} or re.fullmatch(r"(?:best|worst)\d+", raw): + return cls({}, raw) + values: dict[str, str] = {} + for item in _split_options(raw): + key, separator, value = item.partition("=") + if not separator: + continue + values[key.strip().lower()] = value.strip().strip("\"'") + return cls(values, values.pop("for", "best")) + + def select(self, streams: list[StreamSpec]) -> list[StreamSpec]: + selected = [stream for stream in streams if self._matches(stream)] + mode = self.selection.lower() + if mode == "all": + return selected + descending = not mode.startswith("worst") + count_text = re.sub(r"^(?:best|worst)", "", mode) + count = int(count_text) if count_text else 1 + groups: dict[str, list[StreamSpec]] = {} + for stream in selected: + language = (stream.language or "").replace("_", "-").split("-", 1)[0].lower() + groups.setdefault(language, []).append(stream) + return [stream for group in groups.values() for stream in sorted(group, key=lambda item: item.bandwidth or 0, reverse=descending)[:count]] + + def _matches(self, stream: StreamSpec) -> bool: + mapping = { + "id": stream.group_id, + "lang": stream.language, + "name": stream.name, + "codecs": stream.codecs, + "res": stream.resolution, + "frame": str(stream.frame_rate) if stream.frame_rate is not None else None, + "ch": stream.channels, + "range": stream.video_range, + "url": stream.url, + "period": stream.period_id, + "role": stream.role, + } + for key, value in self.values.items(): + if key in mapping and (mapping[key] is None or not re.search(value, mapping[key] or "")): + return False + segments = stream.segments_count + if "segsmin" in self.values and segments <= int(self.values["segsmin"]): + return False + if "segsmax" in self.values and segments >= int(self.values["segsmax"]): + return False + if "bwmin" in self.values and (stream.bandwidth or 0) < int(self.values["bwmin"]) * 1000: + return False + if "bwmax" in self.values and (stream.bandwidth or 0) > int(self.values["bwmax"]) * 1000: + return False + duration = stream.playlist.total_duration if stream.playlist else 0 + if "plistdurmin" in self.values and duration < parse_duration(self.values["plistdurmin"]): + return False + if "plistdurmax" in self.values and duration > parse_duration(self.values["plistdurmax"]): + return False + return True + + +def apply_drop(streams: list[StreamSpec], stream_filter: StreamFilter | None) -> list[StreamSpec]: + if not stream_filter: + return streams + drops = {id(stream) for stream in stream_filter.select(streams)} + return [stream for stream in streams if id(stream) not in drops] + + +def apply_custom_range(streams: list[StreamSpec], value: str | None) -> None: + if not value: + return + left, separator, right = value.partition("-") + if not separator: + raise ValueError("--custom-range must use START-END syntax") + by_time = ":" in value + start = parse_duration(left) if by_time and left else float(left or 0) + end = parse_duration(right) if by_time and right else float("inf") + for stream in streams: + if not stream.playlist: + continue + offset = 0.0 + skipped = 0.0 + for part in stream.playlist.media_parts: + original = part.media_segments + keep: list[MediaSegment] = [] + for segment in original: + match = start <= offset <= end if by_time else start <= segment.index <= end + if match: + keep.append(segment) + elif not keep: + skipped += segment.duration + offset += segment.duration + part.media_segments = keep + stream.playlist.media_parts = [part for part in stream.playlist.media_parts if part.media_segments] + stream.skipped_duration = skipped + + +def clean_ads(streams: list[StreamSpec], patterns: list[str]) -> None: + regexes = [re.compile(pattern) for pattern in patterns] + if not regexes: + return + for stream in streams: + if not stream.playlist: + continue + stream.playlist.media_parts = [ + type(part)([segment for segment in part.media_segments if not any(regex.search(segment.url) for regex in regexes)]) + for part in stream.playlist.media_parts + ] + stream.playlist.media_parts = [part for part in stream.playlist.media_parts if part.media_segments] + if stream.playlist.media_init and any(regex.search(stream.playlist.media_init.url) for regex in regexes): + stream.playlist.media_init = None + + +def automatic_selection(streams: list[StreamSpec], sub_only: bool = False) -> list[StreamSpec]: + video = [stream for stream in streams if stream.media_type is None] + audio = [stream for stream in streams if stream.media_type is MediaType.AUDIO] + subtitles = [stream for stream in streams if stream.media_type is MediaType.SUBTITLES] + if sub_only: + return subtitles + chosen: list[StreamSpec] = [max(video, key=lambda stream: stream.bandwidth or 0)] if video else [] + audio_languages = {stream.language or "" for stream in audio} + for language in audio_languages: + candidates = [stream for stream in audio if (stream.language or "") == language] + chosen.append(max(candidates, key=lambda stream: stream.bandwidth or 0)) + return chosen + subtitles + + +def _split_options(value: str) -> list[str]: + items: list[str] = [] + current: list[str] = [] + quote = "" + escaped = False + for character in value: + if escaped: + current.append(character) + escaped = False + elif character == "\\": + escaped = True + elif character in "\"'": + quote = "" if quote == character else character if not quote else quote + current.append(character) + elif character == ":" and not quote: + items.append("".join(current)) + current = [] + else: + current.append(character) + items.append("".join(current)) + return items diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/subtitles.py b/N_m3u8DL_py/src/n_m3u8dl_py/subtitles.py new file mode 100644 index 0000000..5f64109 --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/subtitles.py @@ -0,0 +1,86 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass + + +TIMING = re.compile(r"(?P[^\s]+)\s+-->\s+(?P[^\s]+)(?P.*)") + + +@dataclass +class Cue: + start: float + end: float + text: str + settings: str = "" + + +def merge_webvtt(files: list[bytes], output_srt: bool) -> str: + cues: list[Cue] = [] + seen: set[tuple[float, float, str]] = set() + for payload in files: + for cue in parse_webvtt(payload.decode("utf-8-sig", errors="replace")): + key = (cue.start, cue.end, cue.text) + if key not in seen: + seen.add(key) + cues.append(cue) + cues.sort(key=lambda cue: (cue.start, cue.end)) + return to_srt(cues) if output_srt else to_vtt(cues) + + +def parse_webvtt(content: str) -> list[Cue]: + lines = content.replace("\r\n", "\n").split("\n") + cues: list[Cue] = [] + index = 0 + while index < len(lines): + line = lines[index].strip() + if " --> " not in line: + index += 1 + continue + match = TIMING.match(line) + if not match: + index += 1 + continue + payload: list[str] = [] + index += 1 + while index < len(lines) and lines[index].strip(): + payload.append(lines[index].strip()) + index += 1 + if payload: + cues.append(Cue(_seconds(match.group("start")), _seconds(match.group("end")), "\n".join(payload), match.group("settings").strip())) + return cues + + +def to_vtt(cues: list[Cue]) -> str: + content = ["WEBVTT", ""] + for cue in cues: + content.extend([f"{_vtt_time(cue.start)} --> {_vtt_time(cue.end)} {cue.settings}".rstrip(), cue.text, ""]) + return "\n".join(content) + "\n" + + +def to_srt(cues: list[Cue]) -> str: + if not cues: + return "1\n00:00:00,000 --> 00:00:01,000\n\n" + content: list[str] = [] + for number, cue in enumerate(cues, 1): + content.extend([str(number), f"{_srt_time(cue.start)} --> {_srt_time(cue.end)}", cue.text, ""]) + return "\n".join(content) + "\n" + + +def _seconds(value: str) -> float: + value = value.replace(",", ".") + if value.endswith("s"): + return float(value[:-1]) + parts = [float(part) for part in value.split(":")] + return parts[-1] + (parts[-2] * 60 if len(parts) >= 2 else 0) + (parts[-3] * 3600 if len(parts) >= 3 else 0) + + +def _vtt_time(value: float) -> str: + hours, remainder = divmod(int(value * 1000), 3_600_000) + minutes, remainder = divmod(remainder, 60_000) + seconds, milliseconds = divmod(remainder, 1000) + return f"{hours:02}:{minutes:02}:{seconds:02}.{milliseconds:03}" + + +def _srt_time(value: float) -> str: + return _vtt_time(value).replace(".", ",") diff --git a/N_m3u8DL_py/src/n_m3u8dl_py/utils.py b/N_m3u8DL_py/src/n_m3u8dl_py/utils.py new file mode 100644 index 0000000..f3ff33c --- /dev/null +++ b/N_m3u8DL_py/src/n_m3u8dl_py/utils.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import base64 +import hashlib +import os +import re +from datetime import timedelta +from pathlib import Path +from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse + + +INVALID_FILENAME = re.compile(r'[<>:"/\\|?*\x00-\x1f]') +VARIABLE_PATTERN = re.compile(r"\$(RepresentationID|Bandwidth|Number|Time)(?:%0?(\d+)([dDxX]))?\$") + + +def combine_url(base_url: str, value: str) -> str: + return urljoin(base_url, value.strip()) + + +def get_attribute(line: str, name: str | None = None) -> str: + payload = line.split(":", 1)[1] if ":" in line else line + if name is None: + return payload.strip() + match = re.search(rf"(?:^|,)\s*{re.escape(name)}=(?:\"([^\"]*)\"|([^,]*))", payload, re.I) + return (match.group(1) if match and match.group(1) is not None else match.group(2) if match else "").strip() + + +def parse_range(value: str) -> tuple[int, int]: + start, end = value.strip().split("-", 1) + first = int(start) + last = int(end) + return first, last - first + 1 + + +def parse_hls_byterange(value: str, previous_end: int | None) -> tuple[int, int]: + length_text, _, offset_text = value.strip().partition("@") + length = int(length_text) + start = int(offset_text) if offset_text else (previous_end or 0) + return start, length + + +def replace_vars(template: str, values: dict[str, object | None]) -> str: + def replacement(match: re.Match[str]) -> str: + name, width, kind = match.groups() + value = values.get(name) + if value is None: + return match.group(0) + if not kind: + return str(value) + integer = int(value) + width_number = int(width or 0) + if kind == "d": + return f"{integer:0{width_number}d}" if width_number else str(integer) + rendered = format(integer, kind.lower()) + rendered = rendered.rjust(width_number, "0") if width_number else rendered + return rendered.upper() if kind == "X" else rendered + + return VARIABLE_PATTERN.sub(replacement, template).replace("$$", "$") + + +def parse_iso8601_duration(value: str | None) -> float: + if not value: + return 0.0 + match = re.fullmatch( + r"P(?:(?P[\d.]+)D)?(?:T(?:(?P[\d.]+)H)?(?:(?P[\d.]+)M)?(?:(?P[\d.]+)S)?)?", + value, + ) + if not match: + raise ValueError(f"Unsupported ISO-8601 duration: {value}") + values = {key: float(number or 0) for key, number in match.groupdict().items()} + return timedelta(days=values["days"], hours=values["hours"], minutes=values["minutes"], seconds=values["seconds"]).total_seconds() + + +def parse_duration(value: str) -> float: + value = value.strip() + if ":" in value: + values = [float(part) for part in value.split(":")] + if len(values) == 2: + return values[0] * 60 + values[1] + if len(values) == 3: + return values[0] * 3600 + values[1] * 60 + values[2] + raise ValueError(f"Invalid duration: {value}") + matches = re.findall(r"([\d.]+)\s*([hms])", value, flags=re.I) + if matches: + factors = {"h": 3600, "m": 60, "s": 1} + return sum(float(amount) * factors[unit.lower()] for amount, unit in matches) + return float(value) + + +def parse_data(value: str | None) -> bytes | None: + if not value: + return None + path = Path(value) + if path.is_file(): + return path.read_bytes() + compact = re.sub(r"\s+", "", value) + if re.fullmatch(r"[0-9a-fA-F]+", compact) and len(compact) % 2 == 0: + return bytes.fromhex(compact) + return base64.b64decode(compact) + + +def valid_filename(value: str, max_bytes: int | None = None) -> str: + value = INVALID_FILENAME.sub("_", value).strip(" .") + return truncate_filename(value or "download", max_bytes) if max_bytes else (value or "download") + + +def truncate_filename(value: str, max_bytes: int = 200) -> str: + encoded = value.encode("utf-8") + if len(encoded) <= max_bytes: + return value + suffix = "_" + hashlib.sha256(encoded).hexdigest()[:12] + budget = max_bytes - len(suffix.encode("utf-8")) + prefix: list[str] = [] + used = 0 + for character in value: + size = len(character.encode("utf-8")) + if used + size > budget: + break + prefix.append(character) + used += size + return "".join(prefix) + suffix + + +def inferred_name(input_value: str) -> str: + parsed = urlparse(input_value) + candidate = Path(parsed.path or input_value).stem or "download" + return valid_filename(candidate, 180) + + +def append_query(source_url: str, target_url: str) -> str: + source_query = parse_qsl(urlparse(source_url).query, keep_blank_values=True) + if not source_query or not target_url.startswith(("http://", "https://")): + return target_url + parsed = urlparse(target_url) + target_query = parse_qsl(parsed.query, keep_blank_values=True) + keys = {key for key, _ in target_query} + merged = target_query + [(key, value) for key, value in source_query if key not in keys] + return urlunparse(parsed._replace(query=urlencode(merged))) + + +def to_jsonable(value: object) -> object: + from dataclasses import asdict, is_dataclass + from enum import Enum + from datetime import datetime + + if is_dataclass(value): + return {key: to_jsonable(item) for key, item in asdict(value).items()} + if isinstance(value, Enum): + return value.value + if isinstance(value, bytes): + return value.hex() + if isinstance(value, datetime): + return value.isoformat() + if isinstance(value, list): + return [to_jsonable(item) for item in value] + if isinstance(value, dict): + return {str(key): to_jsonable(item) for key, item in value.items()} + return value + + +def remove_file(path: Path) -> None: + try: + path.unlink() + except FileNotFoundError: + pass + + +def files_equal_path(left: str, right: str) -> bool: + return os.path.normcase(os.path.abspath(left)) == os.path.normcase(os.path.abspath(right)) diff --git a/N_m3u8DL_py/tests/test_api.py b/N_m3u8DL_py/tests/test_api.py new file mode 100644 index 0000000..c8a9e92 --- /dev/null +++ b/N_m3u8DL_py/tests/test_api.py @@ -0,0 +1,142 @@ +from __future__ import annotations + +from pathlib import Path + +import pytest + +from n_m3u8dl_py import ( + DownloadCancelledError, + DownloadEventKind, + DownloadRequest, + N_m3u8DL, + SelectionError, + TrackSelection, +) + + +def test_public_api_inspects_dash_with_stable_track_ids() -> None: + root = Path(__file__).parents[2] + manifest = root / "src/N_m3u8DL-RE.Tests/Resources/Dash/Manifest_DuplicateSegments.mpd" + info = N_m3u8DL().inspect(manifest) + + assert info.protocol == "DASH" + assert info.tracks[0].track_id == "dash:0" + assert info.tracks[0].segments_count == 3 + assert info.tracks[0].duration == 6 + + +def test_api_downloads_hls_with_default_auto_selection_and_events(tmp_path: Path) -> None: + (tmp_path / "one.ts").write_bytes(b"one") + (tmp_path / "two.ts").write_bytes(b"two") + manifest = tmp_path / "movie.m3u8" + manifest.write_text("#EXTM3U\n#EXTINF:1,\none.ts\n#EXTINF:1,\ntwo.ts\n#EXT-X-ENDLIST\n") + events = [] + client = N_m3u8DL() + + info = client.inspect(manifest, on_event=events.append) + result = client.download( + info, + DownloadRequest( + output_dir=tmp_path / "output", + temporary_dir=tmp_path / "temp", + file_name="movie", + binary_merge=True, + ), + on_event=events.append, + ) + + assert result.selected_track_ids == ("hls:0",) + assert result.files[0].path.read_bytes() == b"onetwo" + assert result.files[0].size == 6 + assert [event.kind for event in events] == [ + DownloadEventKind.INSPECT_STARTED, + DownloadEventKind.INSPECT_COMPLETED, + DownloadEventKind.DOWNLOAD_STARTED, + DownloadEventKind.PLAYLIST_LOADED, + DownloadEventKind.TRACK_STARTED, + DownloadEventKind.SEGMENT_COMPLETED, + DownloadEventKind.SEGMENT_COMPLETED, + DownloadEventKind.TRACK_COMPLETED, + DownloadEventKind.DOWNLOAD_COMPLETED, + ] + segment_events = [event for event in events if event.kind is DownloadEventKind.SEGMENT_COMPLETED] + assert [(event.completed_segments, event.total_segments) for event in segment_events] == [(1, 2), (2, 2)] + + +def test_api_download_url_is_one_call_convenience(tmp_path: Path) -> None: + (tmp_path / "one.ts").write_bytes(b"one") + manifest = tmp_path / "movie.m3u8" + manifest.write_text("#EXTM3U\n#EXTINF:1,\none.ts\n#EXT-X-ENDLIST\n") + + result = N_m3u8DL().download_url( + manifest, + DownloadRequest(output_dir=tmp_path / "output", temporary_dir=tmp_path / "temp", file_name="one-call", binary_merge=True), + ) + + assert result.files[0].path.read_bytes() == b"one" + + +def test_api_downloads_only_explicit_track_id(tmp_path: Path) -> None: + (tmp_path / "video.ts").write_bytes(b"video") + (tmp_path / "audio.ts").write_bytes(b"audio") + (tmp_path / "video.m3u8").write_text("#EXTM3U\n#EXTINF:1,\nvideo.ts\n#EXT-X-ENDLIST\n") + (tmp_path / "audio.m3u8").write_text("#EXTM3U\n#EXTINF:1,\naudio.ts\n#EXT-X-ENDLIST\n") + manifest = tmp_path / "master.m3u8" + manifest.write_text( + "#EXTM3U\n" + '#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",NAME="English",LANGUAGE="en",URI="audio.m3u8"\n' + '#EXT-X-STREAM-INF:BANDWIDTH=1000,RESOLUTION=640x360,AUDIO="aud"\n' + "video.m3u8\n" + ) + client = N_m3u8DL() + info = client.inspect(manifest) + video = next(track for track in info.tracks if track.resolution == "640x360") + + result = client.download( + info, + DownloadRequest( + output_dir=tmp_path / "output", + temporary_dir=tmp_path / "temp", + file_name="video-only", + binary_merge=True, + selection=TrackSelection(track_ids=(video.track_id,)), + ), + ) + + assert result.selected_track_ids == (video.track_id,) + assert len(result.files) == 1 + assert result.files[0].path.read_bytes() == b"video" + + +def test_api_rejects_unknown_track_id(tmp_path: Path) -> None: + (tmp_path / "one.ts").write_bytes(b"one") + manifest = tmp_path / "movie.m3u8" + manifest.write_text("#EXTM3U\n#EXTINF:1,\none.ts\n#EXT-X-ENDLIST\n") + client = N_m3u8DL() + info = client.inspect(manifest) + + with pytest.raises(SelectionError, match="Unknown track ID"): + client.download(info, DownloadRequest(selection=TrackSelection(track_ids=("hls:404",)))) + + +def test_api_event_callback_can_cancel_download(tmp_path: Path) -> None: + (tmp_path / "one.ts").write_bytes(b"one") + manifest = tmp_path / "movie.m3u8" + manifest.write_text("#EXTM3U\n#EXTINF:1,\none.ts\n#EXT-X-ENDLIST\n") + events = [] + + def on_event(event): + events.append(event) + return False if event.kind is DownloadEventKind.TRACK_STARTED else None + + client = N_m3u8DL() + info = client.inspect(manifest, on_event=on_event) + with pytest.raises(DownloadCancelledError): + client.download( + info, + DownloadRequest(output_dir=tmp_path / "output", temporary_dir=tmp_path / "temp", file_name="cancelled"), + on_event=on_event, + ) + + assert events[-1].kind is DownloadEventKind.CANCELLED + assert not (tmp_path / "output" / "cancelled.ts").exists() diff --git a/N_m3u8DL_py/tests/test_cli.py b/N_m3u8DL_py/tests/test_cli.py new file mode 100644 index 0000000..6aaf221 --- /dev/null +++ b/N_m3u8DL_py/tests/test_cli.py @@ -0,0 +1,18 @@ +from n_m3u8dl_py.cli import build_parser + + +def test_cli_accepts_compatibility_options() -> None: + args = build_parser().parse_args( + [ + "input.m3u8", + "--urlprocessor-args", + "token=value", + "--no-date-info", + "--mp4-real-time-decryption", + "--live-fix-vtt-by-audio", + ] + ) + assert args.urlprocessor_args == "token=value" + assert args.no_date_info + assert args.mp4_real_time_decryption + assert args.live_fix_vtt_by_audio diff --git a/N_m3u8DL_py/tests/test_dash.py b/N_m3u8DL_py/tests/test_dash.py new file mode 100644 index 0000000..ca19208 --- /dev/null +++ b/N_m3u8DL_py/tests/test_dash.py @@ -0,0 +1,26 @@ +from pathlib import Path + +from n_m3u8dl_py.dash import DASHExtractor + + +ROOT = Path(__file__).parents[2] + + +def test_dash_resource_parses_expected_stream_shape() -> None: + path = ROOT / "src/N_m3u8DL-RE.Tests/Resources/Dash/Manifest_1080p.mpd" + streams = DASHExtractor(path.resolve().as_uri()).extract(path.read_text()) + assert len(streams) == 23 + first = streams[0] + assert first.bandwidth == 386437 + assert first.extension == "m4s" + assert first.playlist is not None + assert first.playlist.total_duration == 736 + assert first.playlist.media_init is not None + assert first.playlist.media_init.url.endswith("/1/init.mp4") + + +def test_dash_removes_duplicate_segment_urls() -> None: + path = ROOT / "src/N_m3u8DL-RE.Tests/Resources/Dash/Manifest_DuplicateSegments.mpd" + stream = DASHExtractor(path.resolve().as_uri()).extract(path.read_text())[0] + assert stream.playlist is not None + assert [segment.url.rsplit("/", 1)[-1] for segment in stream.playlist.segments] == ["seg-1.m4s", "seg-2.m4s", "seg-3.m4s"] diff --git a/N_m3u8DL_py/tests/test_hls.py b/N_m3u8DL_py/tests/test_hls.py new file mode 100644 index 0000000..145ea15 --- /dev/null +++ b/N_m3u8DL_py/tests/test_hls.py @@ -0,0 +1,127 @@ +from __future__ import annotations + +import contextlib +import functools +import http.server +from pathlib import Path +import shutil +import subprocess +import threading + +from n_m3u8dl_py.cli import main +from n_m3u8dl_py.hls import HLSExtractor +from n_m3u8dl_py.http import HttpClient + + +def test_hls_master_and_media_playlist(tmp_path: Path) -> None: + (tmp_path / "video.m3u8").write_text("#EXTM3U\n#EXTINF:2,\na.ts\n#EXT-X-ENDLIST\n") + master = """#EXTM3U +#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="aud",NAME="English",LANGUAGE="en",URI="audio.m3u8" +#EXT-X-STREAM-INF:BANDWIDTH=1000,RESOLUTION=640x360,AUDIO="aud" +video.m3u8 +""" + extractor = HLSExtractor((tmp_path / "master.m3u8").as_uri(), HttpClient()) + streams = extractor.extract(master) + assert len(streams) == 2 + video = next(stream for stream in streams if stream.resolution == "640x360") + extractor.fetch_playlists([video]) + assert video.playlist is not None + assert video.playlist.segments[0].url.endswith("/a.ts") + + +def test_local_hls_download_and_binary_merge(tmp_path: Path) -> None: + (tmp_path / "one.ts").write_bytes(b"first") + (tmp_path / "two.ts").write_bytes(b"second") + manifest = tmp_path / "test.m3u8" + manifest.write_text("#EXTM3U\n#EXTINF:1,\none.ts\n#EXTINF:1,\ntwo.ts\n#EXT-X-ENDLIST\n") + output = tmp_path / "out" + assert main([str(manifest), "--auto-select", "--binary-merge", "--save-name", "joined", "--save-dir", str(output), "--tmp-dir", str(tmp_path / "tmp")]) == 0 + assert (output / "joined.ts").read_bytes() == b"firstsecond" + + +def test_local_hls_ffmpeg_merge_remuxes_ts_to_mp4(tmp_path: Path) -> None: + (tmp_path / "one.ts").write_bytes(b"first") + manifest = tmp_path / "test.m3u8" + manifest.write_text("#EXTM3U\n#EXTINF:1,\none.ts\n#EXT-X-ENDLIST\n") + ffmpeg = tmp_path / "fake-ffmpeg" + ffmpeg.write_text("#!/bin/sh\nfor argument do output=$argument; done\nprintf remuxed > \"$output\"\n", encoding="utf-8") + ffmpeg.chmod(0o755) + output = tmp_path / "out" + + assert main([ + str(manifest), + "--auto-select", + "--ffmpeg-binary-path", + str(ffmpeg), + "--save-name", + "episode", + "--save-dir", + str(output), + "--tmp-dir", + str(tmp_path / "tmp"), + ]) == 0 + + assert (output / "episode.mp4").read_bytes() == b"remuxed" + assert not (output / "episode.ts").exists() + + +def test_local_hls_save_pattern(tmp_path: Path) -> None: + (tmp_path / "one.ts").write_bytes(b"first") + manifest = tmp_path / "test.m3u8" + manifest.write_text("#EXTM3U\n#EXTINF:1,\none.ts\n#EXT-X-ENDLIST\n") + output = tmp_path / "out" + assert main([str(manifest), "--auto-select", "--binary-merge", "--save-name", "episode", "--save-pattern", "_.", "--save-dir", str(output), "--tmp-dir", str(tmp_path / "tmp")]) == 0 + assert (output / "episode_video.ts").read_bytes() == b"first" + + +def test_local_hls_file_uri_decodes_spaces(tmp_path: Path) -> None: + source = tmp_path / "directory with spaces" + source.mkdir() + (source / "one.ts").write_bytes(b"space-safe") + manifest = source / "test.m3u8" + manifest.write_text("#EXTM3U\n#EXTINF:1,\none.ts\n#EXT-X-ENDLIST\n") + output = tmp_path / "out" + assert main([str(manifest), "--auto-select", "--binary-merge", "--save-name", "spaced", "--save-dir", str(output), "--tmp-dir", str(tmp_path / "tmp")]) == 0 + assert (output / "spaced.ts").read_bytes() == b"space-safe" + + +def test_http_hls_download(tmp_path: Path) -> None: + (tmp_path / "one.ts").write_bytes(b"served-over-http") + (tmp_path / "test.m3u8").write_text("#EXTM3U\n#EXTINF:1,\none.ts\n#EXT-X-ENDLIST\n") + output = tmp_path / "out" + with _http_server(tmp_path) as base_url: + assert main([f"{base_url}/test.m3u8", "--no-use-system-proxy", "--auto-select", "--binary-merge", "--save-name", "http", "--save-dir", str(output), "--tmp-dir", str(tmp_path / "tmp")]) == 0 + assert (output / "http.ts").read_bytes() == b"served-over-http" + + +def test_local_hls_aes128_download(tmp_path: Path) -> None: + key = bytes(range(16)) + iv = bytes(16) + plaintext = b"encrypted payload" + padding = 16 - len(plaintext) % 16 + ciphertext = subprocess.run( + [shutil.which("openssl") or "openssl", "enc", "-aes-128-cbc", "-K", key.hex(), "-iv", iv.hex(), "-nopad"], + input=plaintext + bytes([padding]) * padding, + capture_output=True, + check=True, + ).stdout + (tmp_path / "key.bin").write_bytes(key) + (tmp_path / "encrypted.ts").write_bytes(ciphertext) + manifest = tmp_path / "encrypted.m3u8" + manifest.write_text("#EXTM3U\n#EXT-X-KEY:METHOD=AES-128,URI=\"key.bin\",IV=0x00000000000000000000000000000000\n#EXTINF:1,\nencrypted.ts\n#EXT-X-ENDLIST\n") + output = tmp_path / "out" + assert main([str(manifest), "--auto-select", "--binary-merge", "--save-name", "plain", "--save-dir", str(output), "--tmp-dir", str(tmp_path / "tmp")]) == 0 + assert (output / "plain.ts").read_bytes() == plaintext + + +@contextlib.contextmanager +def _http_server(directory: Path): + handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(directory)) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + thread.join() diff --git a/N_m3u8DL_py/tests/test_http_socks5.py b/N_m3u8DL_py/tests/test_http_socks5.py new file mode 100644 index 0000000..ede4ba4 --- /dev/null +++ b/N_m3u8DL_py/tests/test_http_socks5.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import contextlib +import functools +import http.server +import socket +import threading +from pathlib import Path + +from urllib.request import Request + +from n_m3u8dl_py.http import HttpClient, _Socks5HTTPSHandler + + +def test_http_client_routes_requests_through_no_auth_socks5_proxy(tmp_path: Path) -> None: + (tmp_path / "payload.txt").write_text("proxied", encoding="utf-8") + with _http_server(tmp_path) as target_url, _socks5_proxy() as (proxy_url, address_types): + target_url = target_url.replace("127.0.0.1", "localhost") + data, resolved, _ = HttpClient(proxy=proxy_url, use_system_proxy=False).get_bytes(f"{target_url}/payload.txt") + + assert data == b"proxied" + assert resolved.endswith("/payload.txt") + assert address_types == [3] + + +def test_socks5_https_handler_uses_the_standard_library_context_contract() -> None: + handler = _Socks5HTTPSHandler("127.0.0.1", 1080) + captured: dict[str, object] = {} + + def do_open(http_class, request, **kwargs): + captured.update(kwargs) + return "opened" + + handler.do_open = do_open # type: ignore[method-assign] + + assert handler.https_open(Request("https://example.test/video.m3u8")) == "opened" + assert captured == {"context": handler._context} + + +@contextlib.contextmanager +def _http_server(directory: Path): + handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(directory)) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + server.server_close() + thread.join() + + +@contextlib.contextmanager +def _socks5_proxy(): + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + listener.bind(("127.0.0.1", 0)) + listener.listen() + listener.settimeout(0.2) + stop = threading.Event() + workers: list[threading.Thread] = [] + address_types: list[int] = [] + + def serve() -> None: + while not stop.is_set(): + try: + client, _ = listener.accept() + except TimeoutError: + continue + except OSError: + if stop.is_set(): + return + raise + worker = threading.Thread(target=_handle_client, args=(client, address_types), daemon=True) + workers.append(worker) + worker.start() + + thread = threading.Thread(target=serve, daemon=True) + thread.start() + try: + yield f"socks5://127.0.0.1:{listener.getsockname()[1]}", address_types + finally: + stop.set() + listener.close() + thread.join() + for worker in workers: + worker.join() + + +def _handle_client(client: socket.socket, address_types: list[int]) -> None: + with client: + assert _receive_exact(client, 2) == b"\x05\x01" + assert _receive_exact(client, 1) == b"\x00" + client.sendall(b"\x05\x00") + header = _receive_exact(client, 4) + assert header[:3] == b"\x05\x01\x00" + address_types.append(header[3]) + if header[3] == 1: + host = socket.inet_ntoa(_receive_exact(client, 4)) + elif header[3] == 3: + host = _receive_exact(client, _receive_exact(client, 1)[0]).decode("idna") + else: + raise AssertionError("Unexpected SOCKS address type") + port = int.from_bytes(_receive_exact(client, 2), "big") + with socket.create_connection((host, port)) as upstream: + client.sendall(b"\x05\x00\x00\x01\x00\x00\x00\x00\x00\x00") + left = threading.Thread(target=_copy, args=(client, upstream), daemon=True) + right = threading.Thread(target=_copy, args=(upstream, client), daemon=True) + left.start() + right.start() + left.join() + right.join() + + +def _copy(source: socket.socket, target: socket.socket) -> None: + try: + while data := source.recv(64 * 1024): + target.sendall(data) + except OSError: + pass + finally: + try: + target.shutdown(socket.SHUT_WR) + except OSError: + pass + + +def _receive_exact(connection: socket.socket, count: int) -> bytes: + chunks: list[bytes] = [] + while count: + chunk = connection.recv(count) + if not chunk: + raise OSError("Socket closed") + chunks.append(chunk) + count -= len(chunk) + return b"".join(chunks) diff --git a/N_m3u8DL_py/tests/test_utils.py b/N_m3u8DL_py/tests/test_utils.py new file mode 100644 index 0000000..c4d8106 --- /dev/null +++ b/N_m3u8DL_py/tests/test_utils.py @@ -0,0 +1,22 @@ +from n_m3u8dl_py.utils import replace_vars, truncate_filename, valid_filename + + +def test_replace_vars_supports_dash_number_formats() -> None: + values = {"RepresentationID": "audio", "Number": 28324} + assert replace_vars("$RepresentationID$/$Number%08x$/$Number%05d$/$Number%08X$", values) == "audio/00006ea4/28324/00006EA4" + + +def test_replace_vars_keeps_unknown_values() -> None: + assert replace_vars("$Number%08x$", {}) == "$Number%08x$" + + +def test_filename_truncation_is_stable_and_unicode_safe() -> None: + value = "中" * 100 + result = truncate_filename(value, 50) + assert len(result.encode("utf-8")) <= 50 + assert "\ufffd" not in result + assert valid_filename("a:b*c") == "a_b_c" + + +def test_replace_vars_supports_upper_and_lower_hex() -> None: + assert replace_vars("$Number%08x$/$Number%08X$", {"Number": 22999}) == "000059d7/000059D7" diff --git a/m3u8downloaderd/.gitignore b/m3u8downloaderd/.gitignore new file mode 100644 index 0000000..80552fb --- /dev/null +++ b/m3u8downloaderd/.gitignore @@ -0,0 +1,3 @@ +__pycache__/ +.pytest_cache/ +*.py[cod] diff --git a/m3u8downloaderd/README.md b/m3u8downloaderd/README.md new file mode 100644 index 0000000..ea58377 --- /dev/null +++ b/m3u8downloaderd/README.md @@ -0,0 +1,35 @@ +# m3u8downloaderd + +`m3u8downloaderd` is an in-memory local-network web interface for the adjacent +`N_m3u8DL_py` package. It requires Python 3.10+ and a system `ffmpeg`. + +Install both local packages into the same environment: + +```bash +python -m pip install -e ./N_m3u8DL_py -e ./m3u8downloaderd +``` + +Start the service: + +```bash +python -m m3u8downloaderd \ + --host 0.0.0.0 \ + --port 8000 \ + --download-root /tmp \ + --max-active-downloads 2 +``` + +The service exposes no authentication. Use it only on a trusted network. The +folder chooser is restricted to `--download-root` and its descendants. +Tasks, progress, history, and settings exist only while the process is running; +restarting the service clears them. Downloaded media files remain on disk. +`--max-active-downloads` sets the initial video concurrency. It can be changed +while the service is running from the Settings tab, with a supported range of +1 to 32 videos. Runtime logs are available in the Log tab and remain in memory +only. + +Run tests from the repository root: + +```bash +PYTHONPATH=N_m3u8DL_py/src:m3u8downloaderd/src python -m pytest m3u8downloaderd/tests +``` diff --git a/m3u8downloaderd/pyproject.toml b/m3u8downloaderd/pyproject.toml new file mode 100644 index 0000000..43c71cb --- /dev/null +++ b/m3u8downloaderd/pyproject.toml @@ -0,0 +1,27 @@ +[build-system] +requires = ["setuptools>=68"] +build-backend = "setuptools.build_meta" + +[project] +name = "m3u8downloaderd" +version = "0.1.0" +description = "Local-network web service for N_m3u8DL-PY downloads" +requires-python = ">=3.10" +dependencies = ["n-m3u8dl-py>=0.2.0"] + +[project.scripts] +m3u8downloaderd = "m3u8downloaderd.__main__:main" + +[tool.setuptools] +package-dir = {"" = "src"} +include-package-data = true + +[tool.setuptools.packages.find] +where = ["src"] + +[tool.setuptools.package-data] +m3u8downloaderd = ["static/*"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-q" diff --git a/m3u8downloaderd/src/m3u8downloaderd/__init__.py b/m3u8downloaderd/src/m3u8downloaderd/__init__.py new file mode 100644 index 0000000..c91cb90 --- /dev/null +++ b/m3u8downloaderd/src/m3u8downloaderd/__init__.py @@ -0,0 +1,3 @@ +"""Durable web front end for N_m3u8DL-PY.""" + +__version__ = "0.1.0" diff --git a/m3u8downloaderd/src/m3u8downloaderd/__main__.py b/m3u8downloaderd/src/m3u8downloaderd/__main__.py new file mode 100644 index 0000000..9767ecd --- /dev/null +++ b/m3u8downloaderd/src/m3u8downloaderd/__main__.py @@ -0,0 +1,16 @@ +from __future__ import annotations + +from .web import build_argument_parser, config_from_args, serve + + +def main() -> None: + parser = build_argument_parser() + args = parser.parse_args() + try: + serve(config_from_args(args)) + except (RuntimeError, ValueError) as error: + parser.error(str(error)) + + +if __name__ == "__main__": + main() diff --git a/m3u8downloaderd/src/m3u8downloaderd/models.py b/m3u8downloaderd/src/m3u8downloaderd/models.py new file mode 100644 index 0000000..dbffd8d --- /dev/null +++ b/m3u8downloaderd/src/m3u8downloaderd/models.py @@ -0,0 +1,156 @@ +"""Input validation and path rules shared by the service layers.""" + +from __future__ import annotations + +import base64 +import binascii +import json +import re +from dataclasses import dataclass +from datetime import date, datetime +from pathlib import Path +from urllib.parse import urlsplit + + +REQUIRED_FIELDS = ("code", "title", "href", "image_src", "m3u8_url", "m3u8_referer") +DATE_TEMPLATE = "YYYY-MM-DD_HH-mm-ss" +LEGACY_DATE_TEMPLATE = "YYYY-MM-DD" +DEFAULT_PROXY = "socks5://192.168.4.100:10808" +MAX_ACTIVE_DOWNLOADS = 32 +_SEPARATORS = re.compile(r"[\\/\x00]") + + +class ValidationError(ValueError): + """Raised when a request cannot become a runnable task.""" + + +@dataclass(frozen=True) +class DownloadItemInput: + code: object + title: str + href: object + image_src: str + m3u8_url: str + m3u8_referer: str + + +def resolve_default_title(template: str, now: datetime | date | None = None) -> str: + """Expand date templates; all other values are literal.""" + if template == DATE_TEMPLATE: + current = now or datetime.now() + if isinstance(current, datetime): + return current.strftime("%Y-%m-%d_%H-%M-%S") + return f"{current.isoformat()}_00-00-00" + if template == LEGACY_DATE_TEMPLATE: + return (now or date.today()).isoformat() + return template + + +def decode_download_items(value: str) -> list[DownloadItemInput]: + """Accept JSON or standard/URL-safe base64 that decodes to UTF-8 JSON.""" + if not isinstance(value, str) or not value.strip(): + raise ValidationError("Download information is required") + payload = value.strip() + try: + decoded = json.loads(payload) + except json.JSONDecodeError: + decoded = _decode_base64_json(payload) + if not isinstance(decoded, list) or not decoded: + raise ValidationError("Download information must be a non-empty JSON list") + items: list[DownloadItemInput] = [] + for index, item in enumerate(decoded, 1): + if not isinstance(item, dict): + raise ValidationError(f"Item {index} must be a JSON object") + missing = [field for field in REQUIRED_FIELDS if field not in item] + if missing: + raise ValidationError(f"Item {index} is missing: {', '.join(missing)}") + title = _nonempty_string(item["title"], index, "title") + image_src = _http_url(item["image_src"], index, "image_src") + m3u8_url = _http_url(item["m3u8_url"], index, "m3u8_url") + referer = _nonempty_string(item["m3u8_referer"], index, "m3u8_referer") + items.append( + DownloadItemInput( + code=item["code"], + title=title, + href=item["href"], + image_src=image_src, + m3u8_url=m3u8_url, + m3u8_referer=referer, + ) + ) + return items + + +def validate_proxy(value: object) -> str: + if value in (None, ""): + return "" + if not isinstance(value, str): + raise ValidationError("Proxy must be a string") + parsed = urlsplit(value.strip()) + if parsed.scheme not in {"http", "https", "socks5", "socks5h"} or not parsed.netloc: + raise ValidationError("Proxy must be an HTTP(S) or SOCKS5 URL") + if parsed.username is not None or parsed.password is not None: + raise ValidationError("Authenticated proxies are not supported") + return value.strip() + + +def validate_max_active_downloads(value: object) -> int: + if isinstance(value, bool): + raise ValidationError("Concurrent downloads must be an integer") + if isinstance(value, str): + value = value.strip() + if not value.isdecimal(): + raise ValidationError("Concurrent downloads must be an integer") + value = int(value) + if not isinstance(value, int): + raise ValidationError("Concurrent downloads must be an integer") + if not 1 <= value <= MAX_ACTIVE_DOWNLOADS: + raise ValidationError(f"Concurrent downloads must be between 1 and {MAX_ACTIVE_DOWNLOADS}") + return value + + +def validate_folder_title(value: object) -> str: + if not isinstance(value, str): + raise ValidationError("Task title must be a string") + value = value.strip() + if not value: + return "" + if value in {".", ".."} or _SEPARATORS.search(value): + raise ValidationError("Task title must be one directory name") + return value + + +def ensure_within_root(value: str | Path, root: Path, *, must_exist: bool = True) -> Path: + root = root.resolve() + candidate = Path(value).expanduser().resolve(strict=False) + try: + candidate.relative_to(root) + except ValueError as error: + raise ValidationError("Directory must be inside the configured download root") from error + if must_exist and (not candidate.exists() or not candidate.is_dir()): + raise ValidationError("Selected directory does not exist") + return candidate + + +def _decode_base64_json(value: str) -> object: + compact = "".join(value.split()) + padded = compact + "=" * (-len(compact) % 4) + try: + raw = base64.b64decode(padded, altchars=b"-_", validate=True) + return json.loads(raw.decode("utf-8")) + except (binascii.Error, UnicodeDecodeError, json.JSONDecodeError) as error: + raise ValidationError("Download information must be JSON or base64-encoded JSON") from error + + +def _nonempty_string(value: object, index: int, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise ValidationError(f"Item {index} field {field} must be a non-empty string") + return value.strip() + + +def _http_url(value: object, index: int, field: str) -> str: + result = _nonempty_string(value, index, field) + parsed = urlsplit(result) + if parsed.scheme not in {"http", "https"} or not parsed.netloc: + raise ValidationError(f"Item {index} field {field} must be an HTTP(S) URL") + return result diff --git a/m3u8downloaderd/src/m3u8downloaderd/static/app.js b/m3u8downloaderd/src/m3u8downloaderd/static/app.js new file mode 100644 index 0000000..4872c1d --- /dev/null +++ b/m3u8downloaderd/src/m3u8downloaderd/static/app.js @@ -0,0 +1,340 @@ +const state = { + bootstrap: null, + tasks: [], + directoryTarget: null, + currentDirectory: null, + validationTimer: null, + openActivities: new Set(), + activeTab: "downloads", + logs: [], + logLevel: "", +}; + +const $ = (selector) => document.querySelector(selector); + +async function api(path, options = {}) { + const response = await fetch(path, { + headers: { "Content-Type": "application/json", ...(options.headers || {}) }, + ...options, + }); + const body = await response.json().catch(() => ({})); + if (!response.ok) throw new Error(body.error || `Request failed (${response.status})`); + return body; +} + +function text(value) { + return value == null || value === "" ? "-" : String(value); +} + +function label(value) { + return String(value || "unknown").replaceAll("_", " "); +} + +function make(tag, className, content) { + const node = document.createElement(tag); + if (className) node.className = className; + if (content != null) node.textContent = content; + return node; +} + +async function refreshBootstrap() { + state.bootstrap = await api("/api/bootstrap"); + state.tasks = state.bootstrap.tasks; + renderTasks(); + if (state.activeTab === "logs") await refreshLogs(); +} + +async function refreshLogs() { + const query = state.logLevel ? `?level=${encodeURIComponent(state.logLevel)}` : ""; + const result = await api(`/api/logs${query}`); + state.logs = result.logs; + renderLogs(); +} + +function renderLogs() { + const list = $("#log-list"); + const empty = $("#empty-logs"); + list.replaceChildren(); + empty.hidden = state.logs.length !== 0; + for (const entry of state.logs) { + const row = make("li", `log-row log-${entry.level}`); + const metadata = [entry.created_at.replace("T", " ").replace("+00:00", " UTC"), entry.level.toUpperCase()]; + if (entry.task_id != null) metadata.push(`Task ${entry.task_id}`); + if (entry.item_id != null) metadata.push(`Video ${entry.item_id}`); + row.append(make("span", "log-meta", metadata.join(" · "))); + row.append(make("span", "log-message", entry.message)); + list.append(row); + } +} + +function renderTasks() { + const list = $("#task-list"); + const empty = $("#empty-state"); + list.querySelectorAll(".task-row[data-task-id] .event-log[open]").forEach((entry) => { + state.openActivities.add(entry.closest(".task-row").dataset.taskId); + }); + list.replaceChildren(); + $("#queue-count").textContent = `${state.tasks.length} ${state.tasks.length === 1 ? "task" : "tasks"}`; + empty.hidden = state.tasks.length !== 0; + for (const task of state.tasks) list.append(renderTask(task)); +} + +function renderTask(task) { + const fragment = $("#task-template").content.cloneNode(true); + const row = fragment.querySelector(".task-row"); + row.dataset.status = task.status; + row.dataset.taskId = String(task.id); + const main = fragment.querySelector(".task-main"); + main.append(make("div", "task-title", task.title || task.base_dir)); + main.append(make("div", "task-meta", task.output_dir)); + main.append(make("span", "task-status", label(task.status))); + + const actions = fragment.querySelector(".task-actions"); + if (["queued", "running"].includes(task.status)) { + const cancel = make("button", "secondary-action", "Cancel"); + cancel.type = "button"; + cancel.addEventListener("click", () => mutate(`/api/tasks/${task.id}/cancel`, "Cancel this task?")); + actions.append(cancel); + } + if (["partial", "failed", "cancelled"].includes(task.status)) { + const retry = make("button", "secondary-action", "Retry task"); + retry.type = "button"; + retry.addEventListener("click", () => mutate(`/api/tasks/${task.id}/retry`)); + actions.append(retry); + } + + const progress = taskProgress(task.items); + const progressRoot = fragment.querySelector(".task-progress"); + const caption = make("div", "progress-caption"); + caption.append(make("span", "", progress.caption)); + caption.append(make("span", "", `${progress.percent}%`)); + const meter = document.createElement("progress"); + meter.max = 100; + meter.value = progress.percent; + progressRoot.append(caption, meter); + + const items = fragment.querySelector(".item-list"); + for (const item of task.items) items.append(renderItem(item)); + + const eventLog = fragment.querySelector(".event-log"); + const eventList = eventLog.querySelector("ol"); + if (!task.events.length) { + state.openActivities.delete(String(task.id)); + eventLog.remove(); + } + else { + eventLog.open = state.openActivities.has(String(task.id)); + eventLog.addEventListener("toggle", () => { + if (eventLog.open) state.openActivities.add(String(task.id)); + else state.openActivities.delete(String(task.id)); + }); + for (const event of task.events) { + const entry = make("li", "", `${event.created_at.replace("T", " ").replace("+00:00", " UTC")} · ${event.message}`); + eventList.append(entry); + } + } + return fragment; +} + +function renderItem(item) { + const row = make("div", "item-row"); + const details = make("div", ""); + details.append(make("div", "item-name", item.title)); + details.append(make("div", "item-detail", item.stage)); + const progress = itemProgress(item); + const progressRoot = make("div", "item-progress"); + const caption = make("div", "progress-caption"); + caption.append(make("span", "", progress.caption)); + caption.append(make("span", "", progress.total ? `${progress.percent}%` : "Waiting")); + const meter = document.createElement("progress"); + meter.max = 100; + if (progress.total) meter.value = progress.percent; + progressRoot.append(caption, meter); + details.append(progressRoot); + if (item.warning) details.append(make("div", "item-detail item-warning", item.warning)); + if (item.error) details.append(make("div", "item-detail item-warning", item.error)); + row.append(details); + const actions = make("div", "item-actions"); + if (["failed", "cancelled", "completed_warning"].includes(item.status)) { + const retry = make("button", "secondary-action", item.status === "completed_warning" ? "Retry cover" : "Retry video"); + retry.type = "button"; + retry.addEventListener("click", () => mutate(`/api/items/${item.id}/retry`)); + actions.append(retry); + } + if (item.output_path) actions.append(make("span", "item-detail", "MP4 ready")); + row.append(actions); + return row; +} + +function taskProgress(items) { + const completed = items.filter((item) => ["completed", "completed_warning"].includes(item.status)).length; + const total = items.length; + return { + percent: Math.round((completed / Math.max(total, 1)) * 100), + caption: `${completed}/${total} videos completed`, + }; +} + +function itemProgress(item) { + const total = item.total_segments || 0; + const completed = item.completed_segments || 0; + if (total) { + return { + total, + percent: Math.min(100, Math.round((completed / total) * 100)), + caption: `${completed}/${total} segments`, + }; + } + if (["completed", "completed_warning"].includes(item.status)) { + return { total: 1, percent: 100, caption: "All segments downloaded" }; + } + return { total: 0, percent: 0, caption: "Waiting for playlist" }; +} + +async function mutate(path, confirmation) { + if (confirmation && !window.confirm(confirmation)) return; + try { + await api(path, { method: "POST", body: "{}" }); + await refreshBootstrap(); + } catch (error) { + window.alert(error.message); + } +} + +function showTaskDialog() { + const modal = $("#task-dialog"); + const settings = state.bootstrap.settings; + $("#task-title").value = state.bootstrap.default_title; + $("#task-directory").value = settings.default_directory; + $("#task-payload").value = ""; + setValidation("", false); + modal.showModal(); +} + +function setValidation(message, valid) { + const status = $("#payload-status"); + status.textContent = message; + status.classList.toggle("is-valid", valid); + status.classList.toggle("is-invalid", Boolean(message) && !valid); + $("#start-task").disabled = !valid; +} + +async function validateTaskPayload() { + const payload = $("#task-payload").value; + if (!payload.trim()) return setValidation("", false); + try { + const result = await api("/api/validate", { method: "POST", body: JSON.stringify({ payload }) }); + setValidation(`${result.count} ${result.count === 1 ? "video" : "videos"} ready`, true); + } catch (error) { + setValidation(error.message, false); + } +} + +function openDirectoryPicker(target) { + state.directoryTarget = target; + const input = target === "task" ? $("#task-directory") : $("#setting-directory"); + loadDirectory(input.value).then(() => $("#directory-dialog").showModal()).catch((error) => window.alert(error.message)); +} + +async function loadDirectory(path) { + const result = await api(`/api/directories?path=${encodeURIComponent(path)}`); + state.currentDirectory = result; + $("#directory-path").textContent = result.path; + $("#directory-up").disabled = !result.parent; + const list = $("#directory-list"); + list.replaceChildren(); + for (const directory of result.directories) { + const button = make("button", "directory-entry", directory.name); + button.type = "button"; + button.addEventListener("click", () => loadDirectory(directory.path)); + list.append(button); + } + if (!result.directories.length) list.append(make("p", "item-detail", "No subfolders.")); +} + +async function submitTask(event) { + event.preventDefault(); + const start = $("#start-task"); + start.disabled = true; + try { + await api("/api/tasks", { + method: "POST", + body: JSON.stringify({ + title: $("#task-title").value, + payload: $("#task-payload").value, + directory: $("#task-directory").value, + }), + }); + $("#task-dialog").close(); + await refreshBootstrap(); + } catch (error) { + setValidation(error.message, false); + } +} + +async function submitSettings(event) { + event.preventDefault(); + const status = $("#settings-status"); + try { + const settings = await api("/api/settings", { + method: "PUT", + body: JSON.stringify({ + default_title_template: $("#setting-title").value, + default_directory: $("#setting-directory").value, + proxy: $("#setting-proxy").value, + max_active_downloads: Number($("#setting-max-downloads").value), + }), + }); + state.bootstrap.settings = settings; + await refreshBootstrap(); + status.textContent = "Saved"; + } catch (error) { + status.textContent = error.message; + } +} + +function switchTab(tab) { + state.activeTab = tab; + document.querySelectorAll(".tab").forEach((button) => button.classList.toggle("is-active", button.dataset.tab === tab)); + document.querySelectorAll(".view").forEach((view) => view.classList.toggle("is-active", view.id === `${tab}-view`)); + if (tab === "logs") refreshLogs().catch((error) => window.alert(error.message)); +} + +function bind() { + $("#add-task").addEventListener("click", showTaskDialog); + document.querySelectorAll(".tab").forEach((button) => button.addEventListener("click", () => switchTab(button.dataset.tab))); + $("#close-task").addEventListener("click", () => $("#task-dialog").close()); + $("#cancel-task").addEventListener("click", () => $("#task-dialog").close()); + $("#task-form").addEventListener("submit", submitTask); + $("#settings-form").addEventListener("submit", submitSettings); + $("#task-payload").addEventListener("input", () => { + window.clearTimeout(state.validationTimer); + state.validationTimer = window.setTimeout(validateTaskPayload, 250); + }); + $("#browse-task").addEventListener("click", () => openDirectoryPicker("task")); + $("#browse-settings").addEventListener("click", () => openDirectoryPicker("settings")); + $("#log-level").addEventListener("change", () => { + state.logLevel = $("#log-level").value; + refreshLogs().catch((error) => window.alert(error.message)); + }); + $("#close-directory").addEventListener("click", () => $("#directory-dialog").close()); + $("#directory-up").addEventListener("click", () => loadDirectory(state.currentDirectory.parent)); + $("#select-directory").addEventListener("click", () => { + const selector = state.directoryTarget === "task" ? "#task-directory" : "#setting-directory"; + $(selector).value = state.currentDirectory.path; + $("#directory-dialog").close(); + }); +} + +async function initialize() { + bind(); + await refreshBootstrap(); + const settings = state.bootstrap.settings; + $("#setting-title").value = settings.default_title_template; + $("#setting-directory").value = settings.default_directory; + $("#setting-proxy").value = settings.proxy; + $("#setting-max-downloads").value = settings.max_active_downloads; + window.setInterval(() => refreshBootstrap().catch(() => {}), 1000); +} + +initialize().catch((error) => window.alert(error.message)); diff --git a/m3u8downloaderd/src/m3u8downloaderd/static/index.html b/m3u8downloaderd/src/m3u8downloaderd/static/index.html new file mode 100644 index 0000000..5a9cd07 --- /dev/null +++ b/m3u8downloaderd/src/m3u8downloaderd/static/index.html @@ -0,0 +1,154 @@ + + + + + + m3u8downloaderd + + + +
+
+ + m3u8downloaderd +
+ + +
+ +
+
+
+
+

Queue

+

Downloads

+
+ 0 tasks +
+
+

No download tasks.

+
+ +
+
+
+

Service

+

Settings

+
+
+
+ + + + +
+ + +
+
+
+ +
+
+
+

Runtime

+

Log

+
+ +
+
    + +
    +
    + + +
    + + + + +

    +
    + + +
    +
    +
    + + + +

    +
    + + +
    +
    +
    + + + + + + diff --git a/m3u8downloaderd/src/m3u8downloaderd/static/styles.css b/m3u8downloaderd/src/m3u8downloaderd/static/styles.css new file mode 100644 index 0000000..9906562 --- /dev/null +++ b/m3u8downloaderd/src/m3u8downloaderd/static/styles.css @@ -0,0 +1,131 @@ +:root { + color-scheme: light; + font-family: Inter, ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif; + background: #f4f7f8; + color: #18282d; + line-height: 1.4; +} + +* { box-sizing: border-box; } +body { margin: 0; min-width: 320px; } +button, input, textarea, select { font: inherit; } +button { cursor: pointer; } +button:disabled { cursor: not-allowed; opacity: .5; } + +.topbar { + min-height: 64px; + display: grid; + grid-template-columns: minmax(210px, 1fr) auto minmax(210px, 1fr); + align-items: center; + gap: 20px; + padding: 10px clamp(18px, 4vw, 64px); + background: #10262b; + color: #f9fcfc; + border-bottom: 3px solid #0ac7a1; +} +.brand { display: inline-flex; align-items: center; gap: 10px; font-weight: 750; letter-spacing: 0; } +.brand-mark { display: inline-flex; width: 25px; height: 25px; padding: 4px; gap: 3px; background: #f45e4c; } +.brand-mark i { display: block; width: 4px; background: #10262b; } +.tabs { display: inline-flex; align-self: stretch; } +.tab { border: 0; padding: 0 17px; color: #bad0d1; background: transparent; border-bottom: 2px solid transparent; } +.tab.is-active { color: #ffffff; border-bottom-color: #f5c85d; } +.topbar > .primary-action { justify-self: end; } + +main { max-width: 1200px; margin: 0 auto; padding: 38px clamp(18px, 4vw, 56px) 70px; } +.view { display: none; } +.view.is-active { display: block; } +.view-heading { display: flex; justify-content: space-between; align-items: end; gap: 24px; margin-bottom: 25px; } +.eyebrow { margin: 0 0 4px; color: #42747d; font-size: .76rem; font-weight: 750; letter-spacing: .08em; text-transform: uppercase; } +h1, h2, p { margin-top: 0; } +h1 { margin-bottom: 0; font-size: 1.55rem; } +h2 { margin-bottom: 0; font-size: 1.15rem; } +.queue-count { color: #49646b; font-size: .9rem; } + +.primary-action, .secondary-action, .icon-button, .close-button { + border: 1px solid transparent; + border-radius: 5px; + min-height: 36px; + padding: 0 14px; + font-weight: 680; +} +.primary-action { background: #0d917c; color: #fff; border-color: #0d917c; } +.primary-action:hover:not(:disabled) { background: #087a69; } +.secondary-action { background: #fff; color: #24434a; border-color: #b9c9cc; } +.secondary-action:hover { background: #edf4f4; } +.icon-button, .close-button { background: transparent; color: #24434a; border-color: #b9c9cc; padding: 0 10px; } +.close-button { color: #ba3629; border-color: #edb9b1; } + +.task-list { display: grid; gap: 10px; } +.task-row { background: #fff; border: 1px solid #d4e0e1; border-left: 4px solid #5a7a81; padding: 18px 20px; border-radius: 6px; } +.task-row[data-status="running"] { border-left-color: #0d917c; } +.task-row[data-status="completed"] { border-left-color: #4a8f55; } +.task-row[data-status="partial"], .task-row[data-status="failed"] { border-left-color: #e36b3e; } +.task-row[data-status="cancelled"] { border-left-color: #7d689f; } +.task-row-header { display: flex; justify-content: space-between; gap: 16px; align-items: flex-start; } +.task-main { min-width: 0; } +.task-title { font-weight: 720; overflow-wrap: anywhere; } +.task-meta { margin-top: 3px; color: #5b747a; font-size: .84rem; overflow-wrap: anywhere; } +.task-status { display: inline-block; margin-top: 8px; padding: 2px 8px; background: #e8f1f1; color: #31535a; border-radius: 3px; font-size: .76rem; font-weight: 720; text-transform: uppercase; } +.task-actions { display: inline-flex; flex-wrap: wrap; justify-content: end; gap: 7px; } +.task-progress { margin-top: 14px; } +progress { width: 100%; height: 7px; accent-color: #0d917c; } +.progress-caption { display: flex; justify-content: space-between; margin-bottom: 5px; color: #557076; font-size: .8rem; } +.item-list { display: grid; margin-top: 16px; border-top: 1px solid #e5ecec; } +.item-row { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 14px; align-items: center; padding: 11px 0; border-bottom: 1px solid #e5ecec; } +.item-row:last-child { border-bottom: 0; } +.item-name { font-size: .92rem; font-weight: 680; overflow-wrap: anywhere; } +.item-detail { color: #60797e; font-size: .78rem; overflow-wrap: anywhere; } +.item-progress { margin-top: 8px; max-width: 620px; } +.item-progress progress { height: 5px; accent-color: #2378a8; } +.item-warning { color: #a3462b; } +.item-actions { display: flex; align-items: center; gap: 8px; } +.item-actions .secondary-action { min-height: 30px; padding: 0 9px; font-size: .78rem; } +.event-log { margin-top: 14px; color: #587177; font-size: .8rem; } +.event-log summary { cursor: pointer; font-weight: 650; } +.event-log ol { padding-left: 19px; margin-bottom: 0; } +.event-log li { margin-top: 4px; overflow-wrap: anywhere; } +.empty-state { padding: 58px 0; color: #668087; text-align: center; border-top: 1px solid #cddadb; border-bottom: 1px solid #cddadb; } + +.settings-form { display: grid; max-width: 720px; gap: 19px; background: #fff; border: 1px solid #d4e0e1; padding: 25px; border-radius: 6px; } +label { display: grid; gap: 7px; color: #345158; font-size: .88rem; font-weight: 700; } +input, textarea, select { width: 100%; border: 1px solid #b9c9cc; border-radius: 4px; background: #fff; color: #18282d; padding: 9px 10px; outline-color: #0d917c; } +textarea { min-height: 220px; resize: vertical; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: .84rem; line-height: 1.45; } +.path-control { display: grid; grid-template-columns: minmax(0, 1fr) auto; gap: 8px; } +.form-actions { display: flex; align-items: center; gap: 13px; } +.form-status, .validation-status { min-height: 1.2em; margin: 0; color: #587177; font-size: .84rem; } +.validation-status.is-invalid { color: #b33c30; } +.validation-status.is-valid { color: #17765d; } +.log-filter { width: 150px; font-size: .78rem; } +.log-filter select { padding: 7px 8px; } +.log-list { display: grid; margin: 0; padding: 0; border-top: 1px solid #cddadb; list-style: none; } +.log-row { display: grid; grid-template-columns: minmax(220px, auto) minmax(0, 1fr); gap: 14px; padding: 10px 3px; border-bottom: 1px solid #dce6e7; font-size: .82rem; } +.log-meta { color: #5b747a; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; overflow-wrap: anywhere; } +.log-message { color: #24434a; overflow-wrap: anywhere; } +.log-warning .log-message { color: #9b4b2d; } +.log-error .log-message { color: #b33c30; } + +.modal { width: min(740px, calc(100vw - 28px)); border: 0; border-radius: 7px; padding: 0; color: #18282d; box-shadow: 0 22px 80px rgb(17 39 44 / .3); } +.modal::backdrop { background: rgb(10 25 29 / .56); } +.modal form, .directory-modal { padding: 22px; } +.task-modal form { display: grid; gap: 16px; } +.modal-header { display: flex; align-items: flex-start; justify-content: space-between; gap: 16px; } +.modal-footer { display: flex; justify-content: end; gap: 9px; padding-top: 4px; } +.directory-modal { min-height: min(540px, calc(100vh - 42px)); } +.directory-path { margin: 15px 0 10px; font-family: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; font-size: .82rem; color: #526f75; overflow-wrap: anywhere; } +.directory-actions { display: flex; justify-content: space-between; gap: 10px; padding-bottom: 12px; border-bottom: 1px solid #d6e2e2; } +.directory-list { display: grid; max-height: 360px; overflow: auto; } +.directory-entry { display: flex; width: 100%; min-height: 40px; align-items: center; border: 0; border-bottom: 1px solid #edf1f1; padding: 0 7px; background: #fff; color: #24434a; text-align: left; overflow-wrap: anywhere; } +.directory-entry:hover { background: #eff7f5; } + +@media (max-width: 680px) { + .topbar { grid-template-columns: 1fr auto; gap: 10px; padding: 10px 16px; } + .tabs { grid-row: 2; grid-column: 1 / -1; min-height: 36px; } + .topbar > .primary-action { grid-column: 2; grid-row: 1; padding: 0 10px; font-size: .84rem; } + .task-row { padding: 15px; } + .task-row-header { display: grid; } + .task-actions { justify-content: start; } + .item-row { grid-template-columns: minmax(0, 1fr); } + .item-actions { justify-content: flex-start; } + .view-heading { align-items: start; } + .log-row { grid-template-columns: minmax(0, 1fr); gap: 3px; } +} diff --git a/m3u8downloaderd/src/m3u8downloaderd/tasks.py b/m3u8downloaderd/src/m3u8downloaderd/tasks.py new file mode 100644 index 0000000..35d17e7 --- /dev/null +++ b/m3u8downloaderd/src/m3u8downloaderd/tasks.py @@ -0,0 +1,290 @@ +"""Thread-safe, process-local task state.""" + +from __future__ import annotations + +import copy +import threading +from datetime import datetime, timezone +from pathlib import Path +from typing import Any, Iterable + +from .models import DATE_TEMPLATE, DEFAULT_PROXY, DownloadItemInput + + +MAX_LOG_ENTRIES = 1_000 +LOG_LEVELS = {"debug", "info", "warning", "error"} + + +def _now() -> str: + return datetime.now(timezone.utc).isoformat() + + +class TaskState: + """Keeps settings and task history only for the current process lifetime.""" + + def __init__(self, download_root: Path, max_active_downloads: int = 2) -> None: + self._lock = threading.RLock() + self._settings = { + "default_title_template": DATE_TEMPLATE, + "default_directory": str(download_root.resolve()), + "proxy": DEFAULT_PROXY, + "max_active_downloads": max_active_downloads, + } + self._tasks: dict[int, dict[str, Any]] = {} + self._logs: list[dict[str, Any]] = [] + self._next_task_id = 1 + self._next_item_id = 1 + self._next_event_id = 1 + self._next_log_id = 1 + + def settings(self) -> dict[str, str | int]: + with self._lock: + return dict(self._settings) + + def update_settings(self, values: dict[str, str | int]) -> None: + with self._lock: + self._settings.update(values) + + def max_active_downloads(self) -> int: + with self._lock: + return int(self._settings["max_active_downloads"]) + + def logs(self, level: str | None = None) -> list[dict[str, Any]]: + normalized = (level or "").lower() + if normalized and normalized not in LOG_LEVELS: + raise ValueError("Unknown log level") + with self._lock: + logs = self._logs if not normalized else [entry for entry in self._logs if entry["level"] == normalized] + return copy.deepcopy(logs) + + def log(self, level: str, message: str, task_id: int | None = None, item_id: int | None = None) -> None: + with self._lock: + self._log_unlocked(level, message, task_id, item_id) + + def create_task( + self, + *, + title: str, + base_dir: Path, + output_dir: Path, + proxy: str, + items: Iterable[DownloadItemInput], + ) -> int: + with self._lock: + task_id = self._next_task_id + self._next_task_id += 1 + task = { + "id": task_id, + "title": title, + "base_dir": str(base_dir), + "output_dir": str(output_dir), + "proxy": proxy, + "status": "queued", + "cancel_requested": False, + "created_at": _now(), + "started_at": None, + "completed_at": None, + "error": None, + "items": [], + "events": [], + } + for position, item in enumerate(items, 1): + item_id = self._next_item_id + self._next_item_id += 1 + task["items"].append( + { + "id": item_id, + "task_id": task_id, + "position": position, + "title": item.title, + "image_src": item.image_src, + "m3u8_url": item.m3u8_url, + "m3u8_referer": item.m3u8_referer, + "status": "queued", + "stage": "Queued", + "completed_segments": 0, + "total_segments": 0, + "message": "", + "error": None, + "warning": None, + "output_path": None, + "cover_path": None, + "started_at": None, + "completed_at": None, + } + ) + self._event_unlocked(task, None, "info", "Task queued") + self._tasks[task_id] = task + return task_id + + def claim_next_item(self) -> tuple[dict[str, Any], dict[str, Any]] | None: + with self._lock: + for task in self._tasks.values(): + if task["status"] == "queued": + task.update(status="running", started_at=_now(), cancel_requested=False) + self._event_unlocked(task, None, "info", "Task started") + if task["status"] != "running" or task["cancel_requested"]: + continue + for item in task["items"]: + if item["status"] != "queued": + continue + item.update(status="running", stage="Preparing", started_at=_now(), message="Preparing download") + self._event_unlocked(task, int(item["id"]), "info", "Video started") + return copy.deepcopy(task), copy.deepcopy(item) + return None + + def get_task(self, task_id: int) -> dict[str, Any] | None: + with self._lock: + task = self._tasks.get(task_id) + return copy.deepcopy(task) if task else None + + def list_tasks(self) -> list[dict[str, Any]]: + with self._lock: + return copy.deepcopy(list(reversed(list(self._tasks.values())))) + + def update_item(self, item_id: int, **values: Any) -> None: + with self._lock: + item = self._find_item_unlocked(item_id) + item.update(values) + + def event(self, task_id: int, item_id: int | None, level: str, message: str) -> None: + with self._lock: + self._event_unlocked(self._tasks[task_id], item_id, level, message) + + def cancel_task(self, task_id: int) -> bool: + with self._lock: + task = self._tasks.get(task_id) + if task is None or task["status"] in {"completed", "partial", "failed", "cancelled"}: + return False + now = _now() + task["cancel_requested"] = True + for item in task["items"]: + if item["status"] == "queued": + item.update(status="cancelled", stage="Cancelled", completed_at=now) + self._event_unlocked(task, None, "info", "Cancellation requested") + self._finalize_task_unlocked(task) + return True + + def cancel_requested(self, task_id: int) -> bool: + with self._lock: + return bool(self._tasks[task_id]["cancel_requested"]) + + def finalize_task_if_ready(self, task_id: int) -> None: + with self._lock: + task = self._tasks[task_id] + self._finalize_task_unlocked(task) + + def retry_task(self, task_id: int) -> bool: + with self._lock: + task = self._tasks.get(task_id) + if task is None or task["status"] == "running": + return False + retryable = [item for item in task["items"] if item["status"] in {"failed", "cancelled", "completed_warning"}] + if not retryable: + return False + for item in retryable: + item.update( + status="queued", + stage="Queued", + completed_segments=0, + total_segments=0, + message="", + error=None, + warning=None, + started_at=None, + completed_at=None, + ) + task.update(status="queued", cancel_requested=False, started_at=None, completed_at=None, error=None) + self._event_unlocked(task, None, "info", "Task retry queued") + return True + + def retry_item(self, item_id: int) -> bool: + with self._lock: + task, item = self._find_task_and_item_unlocked(item_id) + if task["status"] == "running" or item["status"] == "completed": + return False + item.update( + status="queued", + stage="Cover retry" if item["status"] == "completed_warning" else "Queued", + completed_segments=0, + total_segments=0, + message="", + error=None, + warning=None, + started_at=None, + completed_at=None, + ) + task.update(status="queued", cancel_requested=False, started_at=None, completed_at=None, error=None) + self._event_unlocked(task, item_id, "info", "Video retry queued") + return True + + def _find_item_unlocked(self, item_id: int) -> dict[str, Any]: + return self._find_task_and_item_unlocked(item_id)[1] + + def _find_task_and_item_unlocked(self, item_id: int) -> tuple[dict[str, Any], dict[str, Any]]: + for task in self._tasks.values(): + for item in task["items"]: + if item["id"] == item_id: + return task, item + raise KeyError(item_id) + + def _event_unlocked(self, task: dict[str, Any], item_id: int | None, level: str, message: str) -> None: + created_at = _now() + task["events"].insert( + 0, + { + "id": self._next_event_id, + "task_id": task["id"], + "item_id": item_id, + "created_at": created_at, + "level": level, + "message": message[:2000], + }, + ) + del task["events"][30:] + self._next_event_id += 1 + self._log_unlocked(level, message, int(task["id"]), item_id, created_at) + + def _log_unlocked( + self, + level: str, + message: str, + task_id: int | None = None, + item_id: int | None = None, + created_at: str | None = None, + ) -> None: + normalized = level.lower() + if normalized not in LOG_LEVELS: + normalized = "info" + self._logs.insert( + 0, + { + "id": self._next_log_id, + "created_at": created_at or _now(), + "level": normalized, + "task_id": task_id, + "item_id": item_id, + "message": message[:2000], + }, + ) + del self._logs[MAX_LOG_ENTRIES:] + self._next_log_id += 1 + + def _finalize_task_unlocked(self, task: dict[str, Any]) -> None: + if task["status"] not in {"queued", "running"}: + return + statuses = {item["status"] for item in task["items"]} + if statuses & {"queued", "running"}: + return + if task["cancel_requested"] and statuses <= {"completed", "completed_warning", "cancelled"}: + status = "cancelled" + elif statuses <= {"completed", "completed_warning"}: + status = "completed" + elif statuses & {"completed", "completed_warning"}: + status = "partial" + elif statuses == {"cancelled"}: + status = "cancelled" + else: + status = "failed" + task.update(status=status, completed_at=_now()) + self._event_unlocked(task, None, "info" if status == "completed" else "warning", f"Task {status}") diff --git a/m3u8downloaderd/src/m3u8downloaderd/web.py b/m3u8downloaderd/src/m3u8downloaderd/web.py new file mode 100644 index 0000000..7162bc0 --- /dev/null +++ b/m3u8downloaderd/src/m3u8downloaderd/web.py @@ -0,0 +1,340 @@ +"""Small dependency-free HTTP server and JSON API for the downloader UI.""" + +from __future__ import annotations + +import argparse +import json +import mimetypes +from dataclasses import dataclass +from http import HTTPStatus +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from typing import Any +from urllib.parse import parse_qs, unquote, urlsplit + +from .models import ( + MAX_ACTIVE_DOWNLOADS, + ValidationError, + decode_download_items, + ensure_within_root, + resolve_default_title, + validate_folder_title, + validate_max_active_downloads, + validate_proxy, +) +from .tasks import TaskState +from .worker import TaskRunner + + +MAX_REQUEST_BYTES = 2 * 1024 * 1024 + + +@dataclass(frozen=True) +class ServiceConfig: + host: str + port: int + download_root: Path + max_active_downloads: int + + +class DownloadService: + def __init__(self, config: ServiceConfig) -> None: + root = config.download_root.expanduser().resolve() + if not root.is_dir(): + raise RuntimeError(f"Download root does not exist or is not a directory: {root}") + self.config = ServiceConfig(config.host, config.port, root, config.max_active_downloads) + self.store = TaskState(root, self.config.max_active_downloads) + self.runner = TaskRunner(self.store) + + def start(self) -> None: + self.runner.start() + + def stop(self) -> None: + self.runner.stop() + + def bootstrap(self) -> dict[str, Any]: + settings = self.store.settings() + return { + "settings": settings, + "default_title": resolve_default_title(str(settings["default_title_template"])), + "download_root": str(self.config.download_root), + "tasks": [_public_task(task) for task in self.store.list_tasks()], + } + + def update_settings(self, value: dict[str, Any]) -> dict[str, str | int]: + current = self.store.settings() + template = value.get("default_title_template", current["default_title_template"]) + directory = value.get("default_directory", current["default_directory"]) + proxy = value.get("proxy", current["proxy"]) + max_active_downloads = value.get("max_active_downloads", current["max_active_downloads"]) + if not isinstance(template, str): + raise ValidationError("Default title must be a string") + selected = ensure_within_root(directory, self.config.download_root) + normalized_proxy = validate_proxy(proxy) + normalized_max_active_downloads = validate_max_active_downloads(max_active_downloads) + settings = { + "default_title_template": template, + "default_directory": str(selected), + "proxy": normalized_proxy, + "max_active_downloads": normalized_max_active_downloads, + } + self.store.update_settings(settings) + self.store.log("info", f"Settings updated: concurrent downloads={normalized_max_active_downloads}") + return settings + + def logs(self, level: str | None) -> dict[str, Any]: + try: + return {"logs": self.store.logs(level)} + except ValueError as error: + raise ValidationError(str(error)) from error + + def directories(self, requested: str | None) -> dict[str, Any]: + selected = ensure_within_root(requested or self.config.download_root, self.config.download_root) + directories: list[dict[str, str]] = [] + for entry in sorted(selected.iterdir(), key=lambda path: path.name.casefold()): + try: + resolved = entry.resolve() + resolved.relative_to(self.config.download_root) + except (OSError, ValueError): + continue + if resolved.is_dir(): + directories.append({"name": entry.name, "path": str(resolved)}) + parent: str | None = None + if selected != self.config.download_root: + candidate = selected.parent.resolve() + try: + candidate.relative_to(self.config.download_root) + except ValueError: + pass + else: + parent = str(candidate) + return {"path": str(selected), "parent": parent, "directories": directories} + + def validate_payload(self, payload: object) -> dict[str, Any]: + items = decode_download_items(payload if isinstance(payload, str) else "") + return {"valid": True, "count": len(items)} + + def create_task(self, value: dict[str, Any]) -> dict[str, Any]: + payload = value.get("payload") + title = validate_folder_title(value.get("title", "")) + items = decode_download_items(payload if isinstance(payload, str) else "") + settings = self.store.settings() + base_dir = ensure_within_root(value.get("directory", settings["default_directory"]), self.config.download_root) + output_dir = base_dir if not title else ensure_within_root(base_dir / title, self.config.download_root, must_exist=False) + output_dir.mkdir(parents=True, exist_ok=True) + task_id = self.store.create_task( + title=title, + base_dir=base_dir, + output_dir=output_dir, + proxy=settings["proxy"], + items=items, + ) + task = self.store.get_task(task_id) + assert task is not None + return _public_task(task) + + def get_task(self, task_id: int) -> dict[str, Any] | None: + task = self.store.get_task(task_id) + return _public_task(task) if task else None + + def list_tasks(self) -> list[dict[str, Any]]: + return [_public_task(task) for task in self.store.list_tasks()] + + +class ServiceRequestHandler(BaseHTTPRequestHandler): + server: "DownloadHTTPServer" + + def do_GET(self) -> None: # noqa: N802 + parsed = urlsplit(self.path) + if parsed.path == "/": + self._serve_static("index.html") + return + if parsed.path.startswith("/static/"): + self._serve_static(unquote(parsed.path.removeprefix("/static/"))) + return + if parsed.path == "/api/bootstrap": + self._json(HTTPStatus.OK, self.server.service.bootstrap()) + return + if parsed.path == "/api/logs": + query = parse_qs(parsed.query) + self._call(lambda: self.server.service.logs(query.get("level", [None])[0])) + return + if parsed.path == "/api/tasks": + self._json(HTTPStatus.OK, {"tasks": self.server.service.list_tasks()}) + return + if parsed.path.startswith("/api/tasks/"): + task_id = _id_from_path(parsed.path, "/api/tasks/") + task = self.server.service.get_task(task_id) if task_id is not None else None + if task is None: + self._error(HTTPStatus.NOT_FOUND, "Task not found") + else: + self._json(HTTPStatus.OK, task) + return + if parsed.path == "/api/directories": + query = parse_qs(parsed.query) + self._call(lambda: self.server.service.directories(query.get("path", [None])[0])) + return + self._error(HTTPStatus.NOT_FOUND, "Not found") + + def do_POST(self) -> None: # noqa: N802 + value = self._request_json() + if value is None: + return + if self.path == "/api/validate": + self._call(lambda: self.server.service.validate_payload(value.get("payload"))) + return + if self.path == "/api/tasks": + self._call(lambda: self.server.service.create_task(value), status=HTTPStatus.CREATED) + return + if self.path.endswith("/cancel") and self.path.startswith("/api/tasks/"): + task_id = _id_from_path(self.path[: -len("/cancel")], "/api/tasks/") + if task_id is None or not self.server.service.store.cancel_task(task_id): + self._error(HTTPStatus.CONFLICT, "Task cannot be cancelled") + else: + task = self.server.service.get_task(task_id) + assert task is not None + self._json(HTTPStatus.OK, task) + return + if self.path.endswith("/retry") and self.path.startswith("/api/tasks/"): + task_id = _id_from_path(self.path[: -len("/retry")], "/api/tasks/") + if task_id is None or not self.server.service.store.retry_task(task_id): + self._error(HTTPStatus.CONFLICT, "Task cannot be retried") + else: + task = self.server.service.get_task(task_id) + assert task is not None + self._json(HTTPStatus.OK, task) + return + if self.path.endswith("/retry") and self.path.startswith("/api/items/"): + item_id = _id_from_path(self.path[: -len("/retry")], "/api/items/") + if item_id is None or not self.server.service.store.retry_item(item_id): + self._error(HTTPStatus.CONFLICT, "Video cannot be retried") + else: + self._json(HTTPStatus.OK, {"ok": True}) + return + self._error(HTTPStatus.NOT_FOUND, "Not found") + + def do_PUT(self) -> None: # noqa: N802 + if self.path != "/api/settings": + self._error(HTTPStatus.NOT_FOUND, "Not found") + return + value = self._request_json() + if value is not None: + self._call(lambda: self.server.service.update_settings(value)) + + def _call(self, callback: Any, *, status: HTTPStatus = HTTPStatus.OK) -> None: + try: + self._json(status, callback()) + except ValidationError as error: + self._error(HTTPStatus.BAD_REQUEST, str(error)) + except OSError as error: + self._error(HTTPStatus.BAD_REQUEST, str(error)) + + def _request_json(self) -> dict[str, Any] | None: + try: + length = int(self.headers.get("Content-Length", "0")) + except ValueError: + self._error(HTTPStatus.BAD_REQUEST, "Invalid Content-Length") + return None + if length <= 0 or length > MAX_REQUEST_BYTES: + self._error(HTTPStatus.REQUEST_ENTITY_TOO_LARGE, "Request body must be between 1 byte and 2 MiB") + return None + try: + value = json.loads(self.rfile.read(length).decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + self._error(HTTPStatus.BAD_REQUEST, "Request body must be UTF-8 JSON") + return None + if not isinstance(value, dict): + self._error(HTTPStatus.BAD_REQUEST, "Request body must be a JSON object") + return None + return value + + def _serve_static(self, relative: str) -> None: + static_root = Path(__file__).with_name("static").resolve() + target = (static_root / relative).resolve() + try: + target.relative_to(static_root) + except ValueError: + self._error(HTTPStatus.NOT_FOUND, "Not found") + return + if not target.is_file(): + self._error(HTTPStatus.NOT_FOUND, "Not found") + return + content_type = mimetypes.guess_type(target.name)[0] or "application/octet-stream" + data = target.read_bytes() + self.send_response(HTTPStatus.OK) + self.send_header("Content-Type", f"{content_type}; charset=utf-8" if content_type.startswith("text/") else content_type) + self.send_header("Content-Length", str(len(data))) + self.send_header("Cache-Control", "no-cache") + self.end_headers() + self.wfile.write(data) + + def _json(self, status: HTTPStatus, value: object) -> None: + data = json.dumps(value, ensure_ascii=False, default=str).encode("utf-8") + self.send_response(status) + self.send_header("Content-Type", "application/json; charset=utf-8") + self.send_header("Content-Length", str(len(data))) + self.send_header("Cache-Control", "no-store") + self.end_headers() + self.wfile.write(data) + + def _error(self, status: HTTPStatus, message: str) -> None: + self._json(status, {"error": message}) + + def log_message(self, format: str, *args: object) -> None: + if not self.path.startswith("/api/bootstrap"): + self.server.service.store.log("debug", f"HTTP {self.command} {self.path}: {format % args}") + + +class DownloadHTTPServer(ThreadingHTTPServer): + def __init__(self, config: ServiceConfig, service: DownloadService) -> None: + super().__init__((config.host, config.port), ServiceRequestHandler) + self.service = service + + +def serve(config: ServiceConfig) -> None: + service = DownloadService(config) + server = DownloadHTTPServer(config, service) + service.start() + try: + server.serve_forever(poll_interval=0.25) + except KeyboardInterrupt: + pass + finally: + server.shutdown() + service.stop() + server.server_close() + + +def build_argument_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Web service for N_m3u8DL-PY") + parser.add_argument("--host", default="0.0.0.0") + parser.add_argument("--port", type=int, default=8000) + parser.add_argument("--download-root", type=Path, default=Path("/tmp")) + parser.add_argument("--max-active-downloads", type=int, default=2) + return parser + + +def config_from_args(args: argparse.Namespace) -> ServiceConfig: + if not 1 <= args.port <= 65535: + raise ValueError("--port must be between 1 and 65535") + if not 1 <= args.max_active_downloads <= MAX_ACTIVE_DOWNLOADS: + raise ValueError(f"--max-active-downloads must be between 1 and {MAX_ACTIVE_DOWNLOADS}") + return ServiceConfig(args.host, args.port, args.download_root, args.max_active_downloads) + + +def _id_from_path(path: str, prefix: str) -> int | None: + value = path.removeprefix(prefix) + try: + return int(value) + except ValueError: + return None + + +def _public_task(task: dict[str, Any] | None) -> dict[str, Any]: + if task is None: + return {} + public = dict(task) + public["items"] = [dict(item) for item in public.get("items", [])] + for item in public["items"]: + item.pop("m3u8_referer", None) + return public diff --git a/m3u8downloaderd/src/m3u8downloaderd/worker.py b/m3u8downloaderd/src/m3u8downloaderd/worker.py new file mode 100644 index 0000000..a35978d --- /dev/null +++ b/m3u8downloaderd/src/m3u8downloaderd/worker.py @@ -0,0 +1,417 @@ +"""Durable task scheduler and media post-processing pipeline.""" + +from __future__ import annotations + +import concurrent.futures +import os +import re +import shutil +import subprocess +import tempfile +import threading +import time +from pathlib import Path +from urllib.error import HTTPError, URLError +from urllib.parse import urlsplit, urlunsplit +from urllib.request import Request + +from n_m3u8dl_py import DownloadCancelledError, DownloadEventKind, DownloadRequest, N_m3u8DL, RequestOptions +from n_m3u8dl_py.http import build_proxy_opener +from n_m3u8dl_py.utils import valid_filename + +from .models import MAX_ACTIVE_DOWNLOADS +from .tasks import TaskState + + +class TaskRunner: + """Runs queued video downloads while keeping HTTP request handling independent.""" + + def __init__(self, store: TaskState, worker_capacity: int = MAX_ACTIVE_DOWNLOADS, ffmpeg: str | None = None) -> None: + self.store = store + self.worker_capacity = max(1, worker_capacity) + self.ffmpeg = ffmpeg or shutil.which("ffmpeg") + if not self.ffmpeg: + raise RuntimeError("ffmpeg is required to run m3u8downloaderd") + self._stop = threading.Event() + self._dispatcher: threading.Thread | None = None + self._executor = concurrent.futures.ThreadPoolExecutor( + max_workers=self.worker_capacity, + thread_name_prefix="m3u8downloaderd-download", + ) + + def start(self) -> None: + if self._dispatcher is None: + self._dispatcher = threading.Thread(target=self._dispatch, name="m3u8downloaderd-dispatch", daemon=True) + self._dispatcher.start() + + def stop(self) -> None: + self._stop.set() + if self._dispatcher is not None: + self._dispatcher.join(timeout=5) + self._executor.shutdown(wait=False, cancel_futures=True) + + def _dispatch(self) -> None: + active: set[concurrent.futures.Future[None]] = set() + while not self._stop.is_set(): + completed = {future for future in active if future.done()} + active.difference_update(completed) + for future in completed: + try: + future.result() + except Exception as error: + self.store.log("error", f"Download worker crashed: {error}") + while len(active) < self.store.max_active_downloads(): + claimed = self.store.claim_next_item() + if claimed is None: + break + task, item = claimed + active.add(self._executor.submit(self._run_item, task, item)) + self._stop.wait(0.2) + + def _run_item(self, task: dict[str, object], item: dict[str, object]) -> None: + item_id = int(item["id"]) + task_id = int(task["id"]) + try: + if item["stage"] == "Cover retry" and item["output_path"]: + self._retry_cover(task, item) + return + self._download_and_finalize(task, item) + except DownloadCancelledError: + self.store.update_item( + item_id, + status="cancelled", + stage="Cancelled", + completed_at=_timestamp(), + message="Cancellation completed", + ) + self.store.event(task_id, item_id, "info", "Video cancelled") + except Exception as error: + self.store.update_item( + item_id, + status="failed", + stage="Failed", + error=str(error), + completed_at=_timestamp(), + message="Download failed", + ) + self.store.event(task_id, item_id, "error", str(error)) + finally: + self.store.finalize_task_if_ready(task_id) + + def _download_and_finalize(self, task: dict[str, object], item: dict[str, object]) -> None: + task_id = int(task["id"]) + item_id = int(item["id"]) + output_dir = Path(str(task["output_dir"])) + output_dir.mkdir(parents=True, exist_ok=True) + progress: dict[str, tuple[int, int]] = {} + + def on_event(event: object) -> bool | None: + if self.store.cancel_requested(task_id): + return False + kind = getattr(event, "kind", None) + track_id = getattr(event, "track_id", None) or "unknown" + if kind is DownloadEventKind.TRACK_STARTED: + progress[track_id] = (0, int(getattr(event, "total_segments", 0) or 0)) + elif kind is DownloadEventKind.SEGMENT_COMPLETED: + progress[track_id] = ( + int(getattr(event, "completed_segments", 0) or 0), + int(getattr(event, "total_segments", 0) or 0), + ) + elif kind is DownloadEventKind.TRACK_COMPLETED and track_id in progress: + done, total = progress[track_id] + progress[track_id] = (total or done, total or done) + if progress: + completed = sum(value[0] for value in progress.values()) + total = sum(value[1] for value in progress.values()) + self.store.update_item( + item_id, + stage="Downloading", + completed_segments=completed, + total_segments=total, + message=getattr(event, "message", "Downloading"), + ) + message = str(getattr(event, "message", "")) + if message: + level = "debug" if kind is DownloadEventKind.SEGMENT_COMPLETED else "info" + self.store.log(level, message, task_id, item_id) + return None + + self.store.log("info", "Inspecting media playlist", task_id, item_id) + self.store.update_item(item_id, stage="Downloading", message="Inspecting media playlist") + options = RequestOptions( + headers={"Referer": str(item["m3u8_referer"])}, + proxy=str(task["proxy"]) or None, + use_system_proxy=False, + ) + with tempfile.TemporaryDirectory(prefix=f".m3u8downloaderd-{task_id}-{item_id}-", dir=output_dir) as temporary_dir: + request = DownloadRequest( + output_dir=output_dir, + temporary_dir=temporary_dir, + file_name=str(item["title"]), + save_pattern=f".source_{item_id}_.", + thread_count=max(1, os.cpu_count() or 1), + ffmpeg_path=self.ffmpeg, + ) + result = N_m3u8DL(options).download_url(str(item["m3u8_url"]), request, on_event=on_event) + if self.store.cancel_requested(task_id): + raise DownloadCancelledError("Download cancelled by user") + self.store.log("info", "Creating MP4", task_id, item_id) + self.store.update_item(item_id, stage="Muxing", message="Creating MP4") + output_path = self._reserve_output_path(output_dir, str(item["title"])) + mux_temp = output_dir / f".{output_path.stem}.{item_id}.mux.tmp.mp4" + try: + self._mux_mp4([file.path for file in result.files], mux_temp) + for source in result.files: + source.path.unlink(missing_ok=True) + self._complete_cover_step(task, item, output_path, mux_temp) + except Exception: + output_path.unlink(missing_ok=True) + mux_temp.unlink(missing_ok=True) + raise + + def _retry_cover(self, task: dict[str, object], item: dict[str, object]) -> None: + item_id = int(item["id"]) + output_path = Path(str(item["output_path"])) + if not output_path.is_file(): + raise FileNotFoundError("The existing MP4 for this cover retry no longer exists") + self.store.log("info", "Retrying cover", int(task["id"]), item_id) + self.store.update_item(item_id, stage="Cover", message="Retrying cover") + cover_path = Path(str(item["cover_path"])) if item["cover_path"] else None + if cover_path is None or not cover_path.is_file(): + cover_path = self._download_cover( + str(item["image_src"]), + output_path.parent, + item_id, + str(task["proxy"]), + task_id=int(task["id"]), + ) + covered_temp = output_path.parent / f".{output_path.stem}.{item_id}.cover.tmp.mp4" + try: + self._attach_cover(output_path, cover_path, covered_temp) + os.replace(covered_temp, output_path) + cover_path.unlink(missing_ok=True) + self.store.update_item( + item_id, + status="completed", + stage="Completed", + warning=None, + cover_path=None, + completed_at=_timestamp(), + message="Video and cover completed", + ) + except Exception as error: + retained_cover = self._retain_cover(cover_path, output_path) + self.store.update_item( + item_id, + status="completed_warning", + stage="Completed with warning", + warning=str(error), + cover_path=str(retained_cover) if retained_cover else None, + completed_at=_timestamp(), + message="Video completed without an embedded cover", + ) + self.store.event(int(task["id"]), item_id, "warning", f"Cover retry failed: {error}") + finally: + covered_temp.unlink(missing_ok=True) + + def _complete_cover_step( + self, + task: dict[str, object], + item: dict[str, object], + output_path: Path, + mux_temp: Path, + ) -> None: + item_id = int(item["id"]) + self.store.log("info", "Downloading cover image", int(task["id"]), item_id) + self.store.update_item(item_id, stage="Cover", message="Downloading cover image") + cover_path: Path | None = None + covered_temp = output_path.parent / f".{output_path.stem}.{item_id}.cover.tmp.mp4" + try: + cover_path = self._download_cover( + str(item["image_src"]), + output_path.parent, + item_id, + str(task["proxy"]), + task_id=int(task["id"]), + ) + self.store.log("info", "Embedding cover image", int(task["id"]), item_id) + self.store.update_item(item_id, stage="Cover", message="Embedding cover image") + self._attach_cover(mux_temp, cover_path, covered_temp) + os.replace(covered_temp, output_path) + mux_temp.unlink(missing_ok=True) + cover_path.unlink(missing_ok=True) + self.store.update_item( + item_id, + status="completed", + stage="Completed", + output_path=str(output_path), + cover_path=None, + completed_at=_timestamp(), + message="Video and cover completed", + ) + self.store.event(int(task["id"]), item_id, "info", f"Saved {output_path.name}") + except Exception as error: + os.replace(mux_temp, output_path) + retained_cover = self._retain_cover(cover_path, output_path) + self.store.update_item( + item_id, + status="completed_warning", + stage="Completed with warning", + output_path=str(output_path), + cover_path=str(retained_cover) if retained_cover else None, + warning=str(error), + completed_at=_timestamp(), + message="Video completed without an embedded cover", + ) + self.store.event(int(task["id"]), item_id, "warning", f"Cover step failed: {error}") + finally: + covered_temp.unlink(missing_ok=True) + + def _mux_mp4(self, source_paths: list[Path], target: Path) -> None: + sources = [path for path in source_paths if path.is_file()] + if not sources: + raise RuntimeError("Downloader did not produce media files") + command = [self.ffmpeg, "-hide_banner", "-loglevel", "error", "-nostdin", "-y"] + for source in sources: + command.extend(["-i", str(source)]) + for index in range(len(sources)): + command.extend(["-map", str(index)]) + command.extend(["-c", "copy"]) + if any(path.suffix.lower() == ".srt" for path in sources): + command.extend(["-c:s", "mov_text"]) + command.extend(["-movflags", "+faststart", str(target)]) + self._run_ffmpeg(command, "MP4 muxing failed") + + def _attach_cover(self, video: Path, cover: Path, target: Path) -> None: + command = [ + self.ffmpeg, + "-hide_banner", + "-loglevel", + "error", + "-nostdin", + "-y", + "-i", + str(video), + "-i", + str(cover), + "-map", + "0", + "-map", + "1:v:0", + "-c", + "copy", + "-c:v:1", + "mjpeg", + "-disposition:v:1", + "attached_pic", + "-movflags", + "+faststart", + str(target), + ] + self._run_ffmpeg(command, "Cover embedding failed") + + def _download_cover(self, url: str, output_dir: Path, item_id: int, proxy: str, *, task_id: int) -> Path: + preferred_url = _preferred_cover_url(url) + if preferred_url != url: + try: + return self._download_cover_url(preferred_url, output_dir, item_id, proxy) + except Exception as error: + self.store.log( + "warning", + f"Preferred s1080 cover URL download failed; falling back to the original URL: {error}", + task_id, + item_id, + ) + return self._download_cover_url(url, output_dir, item_id, proxy) + + def _download_cover_url(self, url: str, output_dir: Path, item_id: int, proxy: str) -> Path: + request = Request(url, headers={"User-Agent": "m3u8downloaderd/0.1"}, method="GET") + opener = build_proxy_opener(proxy or None, use_system_proxy=False) + for attempt in range(4): + try: + with opener.open(request, timeout=100) as response: + content_type = response.headers.get_content_type() + data = _read_limited(response, 25 * 1024 * 1024) + break + except (HTTPError, URLError, TimeoutError, OSError): + if attempt == 3: + raise + time.sleep(attempt + 1) + extension = _image_extension(data, content_type, url) + path = output_dir / f".m3u8downloaderd-{item_id}.cover{extension}" + path.write_bytes(data) + return path + + @staticmethod + def _retain_cover(cover_path: Path | None, output_path: Path) -> Path | None: + if cover_path is None or not cover_path.is_file(): + return None + retained = output_path.with_name(f"{output_path.stem}.cover{cover_path.suffix}") + if retained.exists(): + retained = output_path.with_name(f"{output_path.stem}.cover_{int(time.time())}{cover_path.suffix}") + cover_path.replace(retained) + return retained + + @staticmethod + def _reserve_output_path(output_dir: Path, title: str) -> Path: + stem = valid_filename(title, 180) + suffix = 0 + while True: + candidate = output_dir / f"{stem}{'' if suffix == 0 else f'_{suffix}'}.mp4" + try: + descriptor = os.open(candidate, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o644) + except FileExistsError: + suffix += 1 + continue + os.close(descriptor) + return candidate + + @staticmethod + def _run_ffmpeg(command: list[str], summary: str) -> None: + result = subprocess.run(command, capture_output=True, text=True) + if result.returncode: + detail = result.stderr.strip()[-1500:] or summary + raise RuntimeError(f"{summary}: {detail}") + + +def _read_limited(response: object, limit: int) -> bytes: + chunks: list[bytes] = [] + remaining = limit + 1 + while remaining: + chunk = response.read(min(1024 * 1024, remaining)) # type: ignore[attr-defined] + if not chunk: + break + chunks.append(chunk) + remaining -= len(chunk) + data = b"".join(chunks) + if len(data) > limit: + raise ValueError("Cover image exceeds the 25 MiB size limit") + return data + + +def _image_extension(data: bytes, content_type: str, url: str) -> str: + if data.startswith(b"\xff\xd8\xff") or content_type == "image/jpeg": + return ".jpg" + if data.startswith(b"\x89PNG\r\n\x1a\n") or content_type == "image/png": + return ".png" + if data.startswith(b"RIFF") and data[8:12] == b"WEBP": + return ".webp" + suffix = Path(urlsplit(url).path).suffix.lower() + return suffix if suffix in {".gif", ".bmp", ".avif"} else ".img" + + +_IMG2_SIZE_PATH = re.compile(r"/img2/s\d+/") + + +def _preferred_cover_url(url: str) -> str: + parts = urlsplit(url) + preferred_path = _IMG2_SIZE_PATH.sub("/img2/s1080/", parts.path) + if preferred_path == parts.path: + return url + return urlunsplit(parts._replace(path=preferred_path)) + + +def _timestamp() -> str: + from datetime import datetime, timezone + + return datetime.now(timezone.utc).isoformat() diff --git a/m3u8downloaderd/tests/conftest.py b/m3u8downloaderd/tests/conftest.py new file mode 100644 index 0000000..bddcfbf --- /dev/null +++ b/m3u8downloaderd/tests/conftest.py @@ -0,0 +1,8 @@ +from __future__ import annotations + +import sys +from pathlib import Path + + +ROOT = Path(__file__).parents[2] +sys.path[:0] = [str(ROOT / "m3u8downloaderd" / "src"), str(ROOT / "N_m3u8DL_py" / "src")] diff --git a/m3u8downloaderd/tests/test_models.py b/m3u8downloaderd/tests/test_models.py new file mode 100644 index 0000000..bcc2275 --- /dev/null +++ b/m3u8downloaderd/tests/test_models.py @@ -0,0 +1,62 @@ +from __future__ import annotations + +import base64 +import json +from datetime import date, datetime + +import pytest + +from m3u8downloaderd.models import DATE_TEMPLATE, DEFAULT_PROXY, MAX_ACTIVE_DOWNLOADS, ValidationError, decode_download_items, ensure_within_root, resolve_default_title, validate_max_active_downloads, validate_proxy + + +def _payload() -> list[dict[str, object]]: + return [ + { + "code": 17, + "title": "Episode 1", + "href": "/episode/1", + "image_src": "https://cdn.example.test/cover.jpg", + "m3u8_url": "https://cdn.example.test/video.m3u8", + "m3u8_referer": "https://example.test/watch/1", + "ignored": True, + } + ] + + +def test_download_items_accept_json_and_url_safe_base64() -> None: + value = json.dumps(_payload()) + encoded = base64.urlsafe_b64encode(value.encode()).decode().rstrip("=") + + for input_value in (value, encoded): + items = decode_download_items(input_value) + assert len(items) == 1 + assert items[0].title == "Episode 1" + assert items[0].code == 17 + + +def test_download_items_reject_non_http_sources() -> None: + payload = _payload() + payload[0]["m3u8_url"] = "file:///etc/passwd" + + with pytest.raises(ValidationError, match="HTTP"): + decode_download_items(json.dumps(payload)) + + +def test_template_proxy_and_root_rules(tmp_path) -> None: + assert resolve_default_title(DATE_TEMPLATE, datetime(2026, 9, 26, 14, 30, 5)) == "2026-09-26_14-30-05" + assert resolve_default_title("YYYY-MM-DD", date(2026, 9, 26)) == "2026-09-26" + assert resolve_default_title("weekly") == "weekly" + assert validate_proxy("") == "" + assert validate_proxy("http://127.0.0.1:7890") == "http://127.0.0.1:7890" + assert validate_proxy(DEFAULT_PROXY) == DEFAULT_PROXY + assert validate_max_active_downloads("2") == 2 + assert validate_max_active_downloads(MAX_ACTIVE_DOWNLOADS) == MAX_ACTIVE_DOWNLOADS + with pytest.raises(ValidationError, match="Authenticated"): + validate_proxy("http://name:secret@127.0.0.1:7890") + with pytest.raises(ValidationError, match="between"): + validate_max_active_downloads(MAX_ACTIVE_DOWNLOADS + 1) + child = tmp_path / "child" + child.mkdir() + assert ensure_within_root(child, tmp_path) == child.resolve() + with pytest.raises(ValidationError, match="inside"): + ensure_within_root(tmp_path.parent, tmp_path) diff --git a/m3u8downloaderd/tests/test_service.py b/m3u8downloaderd/tests/test_service.py new file mode 100644 index 0000000..65878d3 --- /dev/null +++ b/m3u8downloaderd/tests/test_service.py @@ -0,0 +1,307 @@ +from __future__ import annotations + +import contextlib +import functools +import http.server +import json +import subprocess +import threading +import time +from pathlib import Path +from urllib.request import Request, urlopen + +from m3u8downloaderd.models import DATE_TEMPLATE, DEFAULT_PROXY, decode_download_items +from m3u8downloaderd.tasks import TaskState +from m3u8downloaderd.web import DownloadHTTPServer, DownloadService, ServiceConfig, build_argument_parser, config_from_args +from m3u8downloaderd.worker import TaskRunner + + +def _item(base_url: str) -> dict[str, str]: + return { + "code": "episode-1", + "title": "episode", + "href": "/episode-1", + "image_src": f"{base_url}/cover.jpg", + "m3u8_url": f"{base_url}/video.m3u8", + "m3u8_referer": "https://example.test/watch/episode-1", + } + + +def test_http_api_validates_and_snapshots_settings(tmp_path: Path) -> None: + config = ServiceConfig("127.0.0.1", 0, tmp_path / "downloads", 1) + config.download_root.mkdir() + service = DownloadService(config) + assert service.bootstrap()["settings"]["proxy"] == DEFAULT_PROXY + assert service.bootstrap()["settings"]["max_active_downloads"] == 1 + server = DownloadHTTPServer(config, service) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + base = f"http://127.0.0.1:{server.server_port}" + invalid = _request(base, "/api/validate", {"payload": "[]"}) + assert invalid[0] == 400 + assert "non-empty" in invalid[1]["error"] + + validation = _request(base, "/api/validate", {"payload": json.dumps([_item("http://127.0.0.1")])}) + assert validation == (200, {"valid": True, "count": 1}) + settings = _request( + base, + "/api/settings", + { + "default_title_template": "YYYY-MM-DD", + "default_directory": str(config.download_root), + "proxy": "http://127.0.0.1:7890", + "max_active_downloads": 2, + }, + method="PUT", + ) + assert settings[0] == 200 + assert settings[1]["max_active_downloads"] == 2 + assert service.store.max_active_downloads() == 2 + task = _request(base, "/api/tasks", {"title": "batch", "directory": str(config.download_root), "payload": json.dumps([_item("http://127.0.0.1")])}) + assert task[0] == 201 + assert task[1]["proxy"] == "http://127.0.0.1:7890" + assert task[1]["output_dir"] == str(config.download_root / "batch") + assert "m3u8_referer" not in task[1]["items"][0] + service.store.log("error", "Expected log entry") + logs = _get(base, "/api/logs?level=error") + assert logs[0] == 200 + assert logs[1]["logs"][0]["message"] == "Expected log entry" + finally: + server.shutdown() + server.server_close() + thread.join() + service.stop() + + +def test_runner_creates_mp4_with_attached_cover(tmp_path: Path) -> None: + source = tmp_path / "source" + source.mkdir() + _make_hls_fixture(source) + root = tmp_path / "downloads" + root.mkdir() + config = ServiceConfig("127.0.0.1", 0, root, 1) + service = DownloadService(config) + service.update_settings({"proxy": ""}) + service.start() + try: + with _http_server(source) as base_url: + task = service.create_task({"title": "collection", "directory": str(root), "payload": json.dumps([_item(base_url)])}) + final = _wait_for_terminal(service, task["id"]) + assert final["status"] == "completed", final + item = final["items"][0] + assert item["status"] == "completed", item + output = Path(item["output_path"]) + assert output == root / "collection" / "episode.mp4" + assert output.is_file() + streams = subprocess.run( + ["ffprobe", "-v", "error", "-show_streams", "-of", "json", str(output)], + capture_output=True, + text=True, + check=True, + ).stdout + assert '"attached_pic": 1' in streams + finally: + service.stop() + + +def test_tasks_and_settings_are_not_retained_after_restart(tmp_path: Path) -> None: + root = tmp_path / "downloads" + root.mkdir() + config = ServiceConfig("127.0.0.1", 0, root, 1) + first = DownloadService(config) + first.update_settings({"default_title_template": "batch", "proxy": ""}) + first.create_task({"title": "batch", "directory": str(root), "payload": json.dumps([_item("http://127.0.0.1")])}) + + restarted = DownloadService(config) + + assert restarted.list_tasks() == [] + assert restarted.bootstrap()["settings"]["default_title_template"] == DATE_TEMPLATE + assert restarted.bootstrap()["settings"]["proxy"] == DEFAULT_PROXY + assert restarted.bootstrap()["settings"]["max_active_downloads"] == 1 + + +def test_runner_runs_multiple_videos_from_one_task_concurrently(tmp_path: Path) -> None: + store = TaskState(tmp_path, max_active_downloads=2) + runner = TaskRunner(store, worker_capacity=2, ffmpeg="ffmpeg") + lock = threading.Lock() + both_started = threading.Event() + release = threading.Event() + active_downloads = 0 + peak_downloads = 0 + + def fake_download(task: dict[str, object], item: dict[str, object]) -> None: + nonlocal active_downloads, peak_downloads + with lock: + active_downloads += 1 + peak_downloads = max(peak_downloads, active_downloads) + if active_downloads == 2: + both_started.set() + release.wait(timeout=5) + store.update_item(int(item["id"]), status="completed", stage="Completed", completed_at="now") + with lock: + active_downloads -= 1 + + runner._download_and_finalize = fake_download # type: ignore[method-assign] + payload = [_item("http://127.0.0.1"), {**_item("http://127.0.0.1"), "code": "episode-2", "title": "episode-2"}] + task_id = store.create_task( + title="batch", + base_dir=tmp_path, + output_dir=tmp_path / "batch", + proxy="", + items=decode_download_items(json.dumps(payload)), + ) + runner.start() + try: + assert both_started.wait(timeout=3) + release.set() + deadline = time.monotonic() + 3 + while time.monotonic() < deadline: + task = store.get_task(task_id) + assert task is not None + if task["status"] == "completed": + break + time.sleep(0.05) + else: + raise AssertionError("Task did not complete") + finally: + release.set() + runner.stop() + + assert peak_downloads == 2 + + +def test_cover_download_prefers_img2_s1080_url(tmp_path: Path) -> None: + store = TaskState(tmp_path) + runner = TaskRunner(store, ffmpeg="ffmpeg") + source_url = "https://cdn.example.test/img2/s720/cover.jpg?token=abc" + expected_url = "https://cdn.example.test/img2/s1080/cover.jpg?token=abc" + downloaded = tmp_path / "cover.jpg" + downloaded.write_bytes(b"cover") + requested_urls: list[str] = [] + + def fake_download(url: str, output_dir: Path, item_id: int, proxy: str) -> Path: + requested_urls.append(url) + return downloaded + + runner._download_cover_url = fake_download # type: ignore[method-assign] + + assert runner._download_cover(source_url, tmp_path, 2, "", task_id=1) == downloaded + assert requested_urls == [expected_url] + assert store.logs("warning") == [] + + +def test_cover_download_falls_back_to_original_img2_url_and_warns(tmp_path: Path) -> None: + store = TaskState(tmp_path) + runner = TaskRunner(store, ffmpeg="ffmpeg") + source_url = "https://cdn.example.test/img2/s720/cover.jpg" + preferred_url = "https://cdn.example.test/img2/s1080/cover.jpg" + downloaded = tmp_path / "cover.jpg" + downloaded.write_bytes(b"cover") + requested_urls: list[str] = [] + + def fake_download(url: str, output_dir: Path, item_id: int, proxy: str) -> Path: + requested_urls.append(url) + if url == preferred_url: + raise OSError("1080p image unavailable") + return downloaded + + runner._download_cover_url = fake_download # type: ignore[method-assign] + + assert runner._download_cover(source_url, tmp_path, 2, "", task_id=1) == downloaded + assert requested_urls == [preferred_url, source_url] + warning = store.logs("warning") + assert len(warning) == 1 + assert warning[0]["task_id"] == 1 + assert warning[0]["item_id"] == 2 + assert "falling back to the original URL" in warning[0]["message"] + + +def test_cover_download_keeps_original_url_without_img2_size_path(tmp_path: Path) -> None: + store = TaskState(tmp_path) + runner = TaskRunner(store, ffmpeg="ffmpeg") + source_url = "https://cdn.example.test/cover.jpg?path=/img2/s720/" + downloaded = tmp_path / "cover.jpg" + downloaded.write_bytes(b"cover") + requested_urls: list[str] = [] + + def fake_download(url: str, output_dir: Path, item_id: int, proxy: str) -> Path: + requested_urls.append(url) + return downloaded + + runner._download_cover_url = fake_download # type: ignore[method-assign] + + assert runner._download_cover(source_url, tmp_path, 2, "", task_id=1) == downloaded + assert requested_urls == [source_url] + + +def test_cli_uses_max_active_downloads() -> None: + args = build_argument_parser().parse_args(["--max-active-downloads", "3"]) + + assert config_from_args(args).max_active_downloads == 3 + + +def _get(base: str, path: str) -> tuple[int, dict[str, object]]: + with urlopen(f"{base}{path}") as response: + return response.status, json.loads(response.read()) + + +def _request(base: str, path: str, payload: dict[str, object], method: str = "POST") -> tuple[int, dict[str, object]]: + request = Request( + f"{base}{path}", + data=json.dumps(payload).encode(), + method=method, + headers={"Content-Type": "application/json"}, + ) + try: + with urlopen(request) as response: + return response.status, json.loads(response.read()) + except Exception as error: + response = error + return response.code, json.loads(response.read()) # type: ignore[attr-defined] + + +def _wait_for_terminal(service: DownloadService, task_id: int, timeout: float = 30) -> dict[str, object]: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + task = service.get_task(task_id) + assert task is not None + if task["status"] in {"completed", "partial", "failed", "cancelled"}: + return task + time.sleep(0.1) + raise AssertionError("Timed out waiting for download task") + + +def _make_hls_fixture(directory: Path) -> None: + subprocess.run( + [ + "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "testsrc2=size=160x90:rate=25:duration=1", + "-f", "lavfi", "-i", "sine=frequency=880:duration=1", + "-shortest", "-c:v", "libx264", "-pix_fmt", "yuv420p", "-c:a", "aac", + "-f", "mpegts", str(directory / "segment.ts"), + ], + check=True, + ) + subprocess.run( + [ + "ffmpeg", "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", "color=c=orange:s=64x64:d=1", "-frames:v", "1", str(directory / "cover.jpg"), + ], + check=True, + ) + (directory / "video.m3u8").write_text("#EXTM3U\n#EXTINF:1,\nsegment.ts\n#EXT-X-ENDLIST\n") + + +@contextlib.contextmanager +def _http_server(directory: Path): + handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=str(directory)) + server = http.server.ThreadingHTTPServer(("127.0.0.1", 0), handler) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + try: + yield f"http://127.0.0.1:{server.server_port}" + finally: + server.shutdown() + server.server_close() + thread.join()