diff --git a/README.md b/README.md index bff1368..537bb70 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,19 @@ # Revids -REST API for generating video from images using [LTX-2.3](https://docs.ltx.video/open-source-model/integration-tools/pytorch-api). +REST API for generating video from images using [LTX-2.3](https://github.com/Lightricks/LTX-2) distilled pipeline. + +## Architecture + +- **FastAPI** REST server with auto-generated OpenAPI docs +- **LTX-2.3 DistilledPipeline** — two-stage video generation (8 steps stage 1, 4 steps stage 2) +- **Async task queue** — submit job, poll status, download when ready +- **SQLite** — job persistence via aiosqlite +- **Single GPU lock** — jobs serialized to avoid VRAM contention ## Quick Start ```bash -# 1. Clone the repo (including submodule) +# 1. Clone the repo (including LTX-2 submodule) git clone --recursive revids cd revids @@ -13,138 +21,227 @@ cd revids python -m venv .venv && source .venv/bin/activate pip install -e libs/LTX-2/packages/ltx-core -e libs/LTX-2/packages/ltx-pipelines -# 3. Install API deps +# 3. Install API dependencies pip install -r requirements.txt -# 4. Download LTX-2.3 model weights +# 4. Download model weights from HuggingFace +## Distilled model checkpoint huggingface-cli download Lightricks/LTX-2.3 \ --include "ltx-2.3-22b-distilled-1.1.safetensors" \ --local-dir models/ -# 5. Download Gemma/GPT-4o text encoder -huggingface-cli download Lightricks/LTX-2 \ - --local-dir models/gpt-4o-5805-ava-gguf-model - -# 6. Download spatial upsampler (optional, for higher-res output) -huggingface-cli download Lightricks/LTX-2 \ - --include "ltx-2.3-22b-spatial-upscaler.safetensors" \ +## Spatial upsampler (required for two-stage pipeline) +huggingface-cli download Lightricks/LTX-2.3 \ + --include "ltx-2.3-spatial-upscaler-x2-1.1.safetensors" \ --local-dir models/ -# 7. Run +## Gemma text encoder +huggingface-cli download google/gemma-3-12b-it-qat-q4_0-unquantized \ + --local-dir models/gemma-3-12b-it-qat-q4_0-unquantized + +# 5. Optimize memory (optional but recommended) +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +# 6. Run uvicorn app.main:app --reload ``` -The API will be available at `http://localhost:8000`. Interactive docs at `http://localhost:8000/docs`. +The API is available at `http://localhost:8000`. Swagger docs at `http://localhost:8000/docs`. ## API Endpoints -### Submit a Job +### Submit Job — `POST /generate` + +Upload an image and generation parameters. Returns a job ID immediately. + ```bash curl -X POST http://localhost:8000/generate \ -F "image=@photo.jpg" \ - -F "prompt=A cinematic pan across the landscape" \ + -F "prompt=A cinematic pan right across a misty mountain range at dawn, golden light breaking through clouds" \ -F "width=768" \ -F "height=512" \ - -F "num_frames=65" \ - -F "fps=24.0" + -F "num_frames=97" \ + -F "fps=24.0" \ + -F "seed=42" ``` -Response: +**Parameters:** + +| Param | Type | Default | Description | +|---|---|---|---| +| `image` | file (required) | — | Input image (PNG, JPG, WebP, etc.) | +| `prompt` | string | `""` | Optional text prompt to guide generation | +| `width` | int | 768 | Output width (must be divisible by 32) | +| `height` | int | 512 | Output height (must be divisible by 32) | +| `num_frames` | int | 65 | Frame count (must be `8n+1`: 9, 17, …, 65, 97, 121, 161, 257) | +| `fps` | float | 24.0 | Frames per second | +| `seed` | int | random | Random seed for reproducibility | + +**Response:** ```json -{"job_id": "abc123def456", "status": "pending"} +{ + "job_id": "abc123def456", + "status": "pending", + "message": "Job submitted. Poll GET /jobs/{job_id} for status." +} ``` -### Check Job Status +### Check Job Status — `GET /jobs/{job_id}` + ```bash curl http://localhost:8000/jobs/abc123def456 ``` -Response: +**Response:** ```json { "job_id": "abc123def456", "status": "completed", - "prompt": "A cinematic pan across the landscape", - "params": {"width": 768, "height": 512, "num_frames": 65, "fps": 24.0}, + "prompt": "A cinematic pan right...", + "params": {"width": 768, "height": 512, "num_frames": 97, "fps": 24.0, "seed": 42}, "error": null, - "created_at": "2025-01-01T00:00:00+00:00", - "completed_at": "2025-01-01T00:01:30+00:00" + "created_at": "2025-01-01T12:00:00+00:00", + "completed_at": "2025-01-01T12:01:30+00:00" } ``` -### Download Video (`.mp4`) +**Status values:** `pending` → `processing` → `completed` or `failed` + +### Download Video — `GET /jobs/{job_id}/download` + +Returns the `.mp4` video file when the job is completed. Returns 404 if not ready. + ```bash curl http://localhost:8000/jobs/abc123def456/download -o output.mp4 ``` -### Delete Job (removes DB record + video file) +### Delete Job — `DELETE /jobs/{job_id}` + +Removes the job record and associated video file. + ```bash curl -X DELETE http://localhost:8000/jobs/abc123def456 ``` -### Health Check +### Health Check — `GET /health` + +Returns GPU availability and device info. + ```bash curl http://localhost:8000/health ``` -Returns GPU availability and device name. -## Generation Constraints +```json +{"status": "ok", "gpu": true, "gpu_name": "NVIDIA A100-SXM4-80GB"} +``` -- **Width/Height**: must be divisible by 32 (min 256, max 2048) -- **num_frames**: must follow `8n+1` pattern (9, 17, ..., 65, 97, 121, 161, 257, ...) -- **fps**: 0 < fps <= 60 +## Frame Count Reference + +Num_frames must follow `8n+1` pattern. Common values: + +| Frames | Duration (24 fps) | Duration (25 fps) | +|---|---|---| +| 65 | ~2.7s | ~2.6s | +| 97 | ~4.0s | ~3.9s | +| 121 | ~5.0s | ~4.8s | +| 161 | ~6.7s | ~6.4s | +| 257 | ~10.7s | ~10.3s | + +## Resolution Reference + +Width and height must be divisible by 32 (min 256, max 2048). + +| Resolution | Aspect Ratio | +|---|---| +| 768×512 | 3:2 landscape | +| 512×768 | 2:3 portrait | +| 704×512 | 4:3 standard | +| 640×640 | 1:1 square | ## Configuration -All settings in `app/config.py` can be overridden via `.env` or env vars prefixed with `REVIDS_`: +All settings can be overridden via `.env` file (placed in project root) or environment variables prefixed with `REVIDS_`. See `app/config.py` for all available options. -| Variable | Default | Description | +| Env Variable | Default | Description | |---|---|---| -| `REVIDS_HOST` | `0.0.0.0` | Bind address | -| `REVIDS_PORT` | `8000` | Port | +| `REVIDS_LTX_DISTILLED_CHECKPOINT` | `models/ltx-2.3-22b-distilled-1.1.safetensors` | Distilled model checkpoint path | +| `REVIDS_LTX_GEMMA_ROOT` | `models/gemma-3-12b-it-qat-q4_0-unquantized` | Gemma text encoder path | +| `REVIDS_LTX_SPATIAL_UPSAMPLER` | `models/ltx-2.3-spatial-upscaler-x2-1.1.safetensors` | Spatial upsampler checkpoint | +| `REVIDS_LTX_QUANTIZATION` | `fp8_cast` | Quantization: `fp8_cast`, `fp8_scaled_mm`, or `""` (bfloat16) | +| `REVIDS_LTX_LORAS` | *(empty)* | LoRA config: `"path1:1.0,path2:0.5"` | +| `REVIDS_MAX_CONCURRENT_JOBS` | `1` | Concurrent GPU jobs | +| `REVIDS_PORT` | `8000` | Server port | | `REVIDS_RELOAD` | `true` | uvicorn auto-reload | -| `REVIDS_LTX_DISTILLED_CHECKPOINT` | `models/ltx-2.3-22b-distilled-1.1.safetensors` | Distilled model path | -| `REVIDS_LTX_GEMMA_ROOT` | `models/gpt-4o-5805-ava-gguf-model` | Gemma text encoder path | -| `REVIDS_LTX_SPATIAL_UPSAMPLER` | *(none)* | Spatial upsampler model path | -| `REVIDS_LTX_QUANTIZATION` | `fp8_cast` | Quantization mode (`fp8_cast`, `fp8_scaled_mm`, or unset for bfloat16) | -| `REVIDS_LTX_LORAS` | *(empty)* | LoRA paths, colon-separated: `"models/lora1.safetensors:1.0,models/lora2.safetensors:0.5"` | -| `REVIDS_MAX_CONCURRENT_JOBS` | `1` | GPU concurrency | + +### LoRA Support + +To load custom or Lightricks LoRAs: + +```bash +# Single LoRA at full strength +export REVIDS_LTX_LORAS="models/my-style.safetensors:1.0" + +# Multiple LoRAs +export REVIDS_LTX_LORAS="models/style.safetensors:1.0,models/motion.safetensors:0.5" +``` + +Lightricks-published LoRAs: +- [IC-LoRA Union Control](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-Union-Control) +- [IC-LoRA Motion Track Control](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-Motion-Track-Control) +- [IC-LoRA LipDub](https://huggingface.co/Lightricks/LTX-2.3-22b-IC-LoRA-LipDub) +- Camera Control LoRAs (dolly in/out, jib up/down, static, etc.) + +## Memory Optimization + +```bash +# Improve CUDA memory allocation fragmentation +export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True + +# Use FP8 quantization (~40% VRAM reduction, minimal quality loss) +export REVIDS_LTX_QUANTIZATION=fp8_cast + +# For Hopper GPUs (H100) with TensorRT-LLM: +# export REVIDS_LTX_QUANTIZATION=fp8_scaled_mm +``` + +## Hardware Requirements + +- **GPU**: NVIDIA with CUDA support +- **VRAM**: ≥ 24 GB (bfloat16), ~14 GB with FP8 quantization +- **Disk**: ~30 GB for all model weights (distilled + upsampler + Gemma encoder) ## Project Structure ``` revids/ app/ - main.py # FastAPI app + routes - config.py # Settings (pydantic-settings) - models.py # Request/response schemas - service.py # LTX pipeline wrapper + job processing - database.py # SQLite job store + __init__.py + main.py # FastAPI app + HTTP routes + config.py # Settings (pydantic-settings, env var support) + models.py # Request/response Pydantic models + service.py # LTX pipeline wrapper, job processing, video encoding + database.py # SQLite job store (aiosqlite) libs/ - LTX-2/ # LTX-2 git submodule + LTX-2/ # LTX-2 git submodule (source for ltx-core/ltx-pipelines) + videos/ # Generated .mp4 output (gitignored) requirements.txt - videos/ # Generated video output (gitignored) + .gitignore ``` -## Hardware Requirements +## Prompting Guide -- NVIDIA GPU with CUDA (>= 24GB VRAM for bfloat16, ~14GB with FP8) -- ~30GB disk for model weights +For best results with LTX-2.3, write detailed, chronological motion descriptions: -## Memory Optimization +1. Start with the main action in one sentence +2. Add specific movement details and gestures +3. Describe appearances precisely +4. Include background and environment +5. Specify camera angle and movement +6. Describe lighting and color palette +7. Keep under 200 words -```bash -export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True -``` +Full guide: [https://ltx.video/blog/how-to-prompt-for-ltx-2](https://ltx.video/blog/how-to-prompt-for-ltx-2) -Set `REVIDS_LTX_QUANTIZATION=fp8_cast` (~40% VRAM reduction) or `fp8_scaled_mm` (Hopper GPUs only). +## License -## LoRA Support - -Configure LoRAs via environment variable: - -```bash -export REVIDS_LTX_LORAS="models/my-style.safetensors:1.0,models/my-motion.safetensors:0.5" -``` - -Or set as a comma-separated list of `path:scale` entries. +See [LTX-2 LICENSE](libs/LTX-2/LICENSE) for model usage terms. diff --git a/app/config.py b/app/config.py index 78e7a14..35056c6 100644 --- a/app/config.py +++ b/app/config.py @@ -12,14 +12,13 @@ class Settings(BaseSettings): port: int = 8000 reload: bool = True - # LTX-2.3 model paths (update after downloading models) + # LTX-2.3 model paths (update after downloading from HuggingFace) ltx_distilled_checkpoint: str = "models/ltx-2.3-22b-distilled-1.1.safetensors" - ltx_gemma_root: str = "models/gpt-4o-5805-ava-gguf-model" - ltx_spatial_upsampler: str | None = None + ltx_gemma_root: str = "models/gemma-3-12b-it-qat-q4_0-unquantized" + ltx_spatial_upsampler: str = "models/ltx-2.3-spatial-upscaler-x2-1.1.safetensors" ltx_device: str = "cuda" ltx_quantization: str = "fp8_cast" - # LoRA paths (list of "path:scale" strings, env: REVIDS_LTX_LORAS="models/my-lora.safetensors:1.0") ltx_loras: list[str] = [] video_output_dir: str = os.path.join(os.path.dirname(os.path.dirname(__file__)), "videos") diff --git a/app/service.py b/app/service.py index a89df7d..f8f3fe7 100644 --- a/app/service.py +++ b/app/service.py @@ -1,18 +1,24 @@ from __future__ import annotations import asyncio +import logging import os import uuid +from collections.abc import Iterator from datetime import datetime, timezone from typing import Optional import torch +from ltx_core.loader import LoraPathStrengthAndSDOps from ltx_pipelines import DistilledPipeline +from ltx_pipelines.utils.args import ImageConditioningInput from PIL import Image from app.config import settings from app.database import JobDB, JobRecord, VALID_FRAME_COUNTS +logger = logging.getLogger(__name__) + class LtxService: def __init__(self, db: JobDB) -> None: @@ -26,7 +32,10 @@ class LtxService: async with self._load_lock: if self.pipeline is not None: return + logger.info("Loading DistilledPipeline...") lora_config = self._parse_lora_paths(settings.ltx_loras) + quantization = self._build_quantization_policy() + self.pipeline = await asyncio.to_thread( DistilledPipeline, distilled_checkpoint_path=settings.ltx_distilled_checkpoint, @@ -34,21 +43,60 @@ class LtxService: spatial_upsampler_path=settings.ltx_spatial_upsampler, loras=lora_config, device=settings.ltx_device, - quantization=settings.ltx_quantization, + quantization=quantization, ) + logger.info("Pipeline loaded successfully") @staticmethod - def _parse_lora_paths(lora_specs: list[str]) -> list[dict]: + def _parse_lora_paths(lora_specs: list[str]) -> list[LoraPathStrengthAndSDOps]: result = [] for spec in lora_specs: if ":" in spec: path, scale = spec.rsplit(":", 1) - result.append({"path": path.strip(), "scale": float(scale.strip())}) + result.append( + LoraPathStrengthAndSDOps( + path=path.strip(), strength=float(scale.strip()), sd_ops=[] + ) + ) else: - result.append({"path": spec.strip(), "scale": 1.0}) + result.append( + LoraPathStrengthAndSDOps(path=spec.strip(), strength=1.0, sd_ops=[]) + ) return result - async def submit_job(self, image: Image.Image, request_params: dict) -> JobRecord: + @staticmethod + def _build_quantization_policy() -> object | None: + q = settings.ltx_quantization + if not q: + return None + if q == "fp8_cast": + from ltx_core.quantization.fp8_cast import ( + TRANSFORMER_LINEAR_DOWNCAST_MAP, + UPCAST_DURING_INFERENCE, + fp8_cast_fuse_rule, + ) + from ltx_core.quantization.policy import QuantizationPolicy + + return QuantizationPolicy( + sd_ops=TRANSFORMER_LINEAR_DOWNCAST_MAP, + module_ops=(UPCAST_DURING_INFERENCE,), + fuse_rule=fp8_cast_fuse_rule, + ) + if q == "fp8_scaled_mm": + from ltx_core.quantization.fp8_scaled_mm import ( + FP8_SCALED_MM_MODULE_OPS, + fp8_scaled_mm_fuse_rule, + ) + from ltx_core.quantization.policy import QuantizationPolicy + + return QuantizationPolicy( + module_ops=FP8_SCALED_MM_MODULE_OPS, fuse_rule=fp8_scaled_mm_fuse_rule + ) + return None + + async def submit_job( + self, image: Image.Image, request_params: dict[str, object] + ) -> JobRecord: job_id = uuid.uuid4().hex[:12] now = datetime.now(timezone.utc).isoformat() prompt = request_params.get("prompt") @@ -75,7 +123,7 @@ class LtxService: "width": width, "height": height, "num_frames": num_frames, - "fps": fps, + "fps": float(fps), "seed": seed, }, created_at=now, @@ -88,56 +136,73 @@ class LtxService: self, job_id: str, image: Image.Image, - params: dict, + params: dict[str, object], ) -> None: async with self._job_semaphore: try: + logger.info(f"Job {job_id}: starting processing") await self.db.update_status(job_id, "processing") await self.load_pipeline() - prompt = params.get("prompt", "") or "" - width = params.get("width", settings.default_width) - height = params.get("height", settings.default_height) - num_frames = params.get("num_frames", settings.default_frames) - fps = params.get("fps", settings.default_fps) - seed = params.get("seed") if params.get("seed") is not None else 42 + prompt: str = params.get("prompt", "") or "" + width: int = params.get("width", settings.default_width) + height: int = params.get("height", settings.default_height) + num_frames: int = params.get("num_frames", settings.default_frames) + fps: float = params.get("fps", settings.default_fps) + seed: int = int(params.get("seed")) if params.get("seed") is not None else 42 + # Save image to temp file (ImageConditioningInput expects a path) if image.mode != "RGB": image = image.convert("RGB") image = image.resize((width, height), Image.LANCZOS) - output = await asyncio.to_thread( + img_path = os.path.join( + settings.video_output_dir, f"{job_id}_input.png" + ) + os.makedirs(settings.video_output_dir, exist_ok=True) + image.save(img_path) + + conditioning = [ + ImageConditioningInput( + path=img_path, frame_idx=0, strength=1.0, crf=33 + ) + ] + + logger.info(f"Job {job_id}: running inference") + result = await asyncio.to_thread( self.pipeline, prompt=prompt, - images=image, + seed=seed, width=width, height=height, num_frames=num_frames, - frame_rate=fps, - seed=seed, + frame_rate=float(fps), + images=conditioning, ) + # result is tuple[Iterator[torch.Tensor], Audio] + video_frames: Iterator[torch.Tensor] = result[0] video_path = os.path.join(self._video_dir, f"{job_id}.mp4") - os.makedirs(self._video_dir, exist_ok=True) - await asyncio.to_thread(self._save_video, output, video_path, fps) + await asyncio.to_thread(self._save_video, video_frames, video_path, fps) + + # Clean up temp input image + if os.path.exists(img_path): + os.remove(img_path) + + logger.info(f"Job {job_id}: completed") await self.db.update_status(job_id, "completed") except Exception as e: + logger.exception(f"Job {job_id}: failed") await self.db.update_status(job_id, "failed", error=str(e)) @staticmethod - def _save_video(output: any, path: str, fps: float) -> None: + def _save_video( + frames_iter: Iterator[torch.Tensor], path: str, fps: float + ) -> None: import imageio.v3 as iio - if isinstance(output, torch.Tensor): - frames = output.cpu().numpy() - elif isinstance(output, dict) and "video" in output: - v = output["video"] - frames = v.cpu().numpy() if isinstance(v, torch.Tensor) else v - else: - frames = output - if isinstance(frames, torch.Tensor): - frames = frames.cpu().numpy() + frames = next(frames_iter).cpu().numpy() if frames.ndim == 4: frames = frames[0]