mteehan/revids
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix API to match actual ltx-pipelines 1.1.5 interface

Browse Source 
- Use DistilledPipeline constructor (not from_config)
- Use ImageConditioningInput namedtuple for image conditioning
- Use LoraPathStrengthAndSDOps for LoRA config
- Use QuantizationPolicy with fp8_cast sd_ops directly
- Update spatial_upsampler to required x2-1.1 model
- Update gemma_root to google/gemma-3-12b-it-qat-q4_0-unquantized
- Handle video output as Iterator[torch.Tensor] from pipeline
- Flesh out README with all model download commands, endpoint docs, env vars, frame/resolution tables
This commit is contained in:
Mike Teehan
2026-06-01 19:37:16 -04:00
parent 2d50679ec0
commit a33a3a593d
3 changed files with 261 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
app/config.py
+3 -4
View File
@@ -12,14 +12,13 @@ class Settings(BaseSettings):
port: int = 8000
reload: bool = True
# LTX-2.3 model paths (update after downloading models)
# LTX-2.3 model paths (update after downloading from HuggingFace)
ltx_distilled_checkpoint: str = "models/ltx-2.3-22b-distilled-1.1.safetensors"
ltx_gemma_root: str = "models/gpt-4o-5805-ava-gguf-model"
ltx_spatial_upsampler: str | None = None
ltx_gemma_root: str = "models/gemma-3-12b-it-qat-q4_0-unquantized"
ltx_spatial_upsampler: str = "models/ltx-2.3-spatial-upscaler-x2-1.1.safetensors"
ltx_device: str = "cuda"
ltx_quantization: str = "fp8_cast"
# LoRA paths (list of "path:scale" strings, env: REVIDS_LTX_LORAS="models/my-lora.safetensors:1.0")
ltx_loras: list[str] = []
video_output_dir: str = os.path.join(os.path.dirname(os.path.dirname(__file__)), "videos")
app/service.py
+94 -29
View File
@@ -1,18 +1,24 @@
from __future__ import annotations
import asyncio
import logging
import os
import uuid
from collections.abc import Iterator
from datetime import datetime, timezone
from typing import Optional
import torch
from ltx_core.loader import LoraPathStrengthAndSDOps
from ltx_pipelines import DistilledPipeline
from ltx_pipelines.utils.args import ImageConditioningInput
from PIL import Image
from app.config import settings
from app.database import JobDB, JobRecord, VALID_FRAME_COUNTS
logger = logging.getLogger(__name__)
class LtxService:
def __init__(self, db: JobDB) -> None:
@@ -26,7 +32,10 @@ class LtxService:
async with self._load_lock:
if self.pipeline is not None:
return
logger.info("Loading DistilledPipeline...")
lora_config = self._parse_lora_paths(settings.ltx_loras)
quantization = self._build_quantization_policy()
self.pipeline = await asyncio.to_thread(
DistilledPipeline,
distilled_checkpoint_path=settings.ltx_distilled_checkpoint,
@@ -34,21 +43,60 @@ class LtxService:
spatial_upsampler_path=settings.ltx_spatial_upsampler,
loras=lora_config,
device=settings.ltx_device,
quantization=settings.ltx_quantization,
quantization=quantization,
)
logger.info("Pipeline loaded successfully")
@staticmethod
def _parse_lora_paths(lora_specs: list[str]) -> list[dict]:
def _parse_lora_paths(lora_specs: list[str]) -> list[LoraPathStrengthAndSDOps]:
result = []
for spec in lora_specs:
if ":" in spec:
path, scale = spec.rsplit(":", 1)
result.append({"path": path.strip(), "scale": float(scale.strip())})
result.append(
LoraPathStrengthAndSDOps(
path=path.strip(), strength=float(scale.strip()), sd_ops=[]
)
)
else:
result.append({"path": spec.strip(), "scale": 1.0})
result.append(
LoraPathStrengthAndSDOps(path=spec.strip(), strength=1.0, sd_ops=[])
)
return result
async def submit_job(self, image: Image.Image, request_params: dict) -> JobRecord:
@staticmethod
def _build_quantization_policy() -> object | None:
q = settings.ltx_quantization
if not q:
return None
if q == "fp8_cast":
from ltx_core.quantization.fp8_cast import (
TRANSFORMER_LINEAR_DOWNCAST_MAP,
UPCAST_DURING_INFERENCE,
fp8_cast_fuse_rule,
)
from ltx_core.quantization.policy import QuantizationPolicy
return QuantizationPolicy(
sd_ops=TRANSFORMER_LINEAR_DOWNCAST_MAP,
module_ops=(UPCAST_DURING_INFERENCE,),
fuse_rule=fp8_cast_fuse_rule,
)
if q == "fp8_scaled_mm":
from ltx_core.quantization.fp8_scaled_mm import (
FP8_SCALED_MM_MODULE_OPS,
fp8_scaled_mm_fuse_rule,
)
from ltx_core.quantization.policy import QuantizationPolicy
return QuantizationPolicy(
module_ops=FP8_SCALED_MM_MODULE_OPS, fuse_rule=fp8_scaled_mm_fuse_rule
)
return None
async def submit_job(
self, image: Image.Image, request_params: dict[str, object]
) -> JobRecord:
job_id = uuid.uuid4().hex[:12]
now = datetime.now(timezone.utc).isoformat()
prompt = request_params.get("prompt")
@@ -75,7 +123,7 @@ class LtxService:
"width": width,
"height": height,
"num_frames": num_frames,
"fps": fps,
"fps": float(fps),
"seed": seed,
},
created_at=now,
@@ -88,56 +136,73 @@ class LtxService:
self,
job_id: str,
image: Image.Image,
params: dict,
params: dict[str, object],
) -> None:
async with self._job_semaphore:
try:
logger.info(f"Job {job_id}: starting processing")
await self.db.update_status(job_id, "processing")
await self.load_pipeline()
prompt = params.get("prompt", "") or ""
width = params.get("width", settings.default_width)
height = params.get("height", settings.default_height)
num_frames = params.get("num_frames", settings.default_frames)
fps = params.get("fps", settings.default_fps)
seed = params.get("seed") if params.get("seed") is not None else 42
prompt: str = params.get("prompt", "") or ""
width: int = params.get("width", settings.default_width)
height: int = params.get("height", settings.default_height)
num_frames: int = params.get("num_frames", settings.default_frames)
fps: float = params.get("fps", settings.default_fps)
seed: int = int(params.get("seed")) if params.get("seed") is not None else 42
# Save image to temp file (ImageConditioningInput expects a path)
if image.mode != "RGB":
image = image.convert("RGB")
image = image.resize((width, height), Image.LANCZOS)
output = await asyncio.to_thread(
img_path = os.path.join(
settings.video_output_dir, f"{job_id}_input.png"
)
os.makedirs(settings.video_output_dir, exist_ok=True)
image.save(img_path)
conditioning = [
ImageConditioningInput(
path=img_path, frame_idx=0, strength=1.0, crf=33
)
]
logger.info(f"Job {job_id}: running inference")
result = await asyncio.to_thread(
self.pipeline,
prompt=prompt,
images=image,
seed=seed,
width=width,
height=height,
num_frames=num_frames,
frame_rate=fps,
seed=seed,
frame_rate=float(fps),
images=conditioning,
)
# result is tuple[Iterator[torch.Tensor], Audio]
video_frames: Iterator[torch.Tensor] = result[0]
video_path = os.path.join(self._video_dir, f"{job_id}.mp4")
os.makedirs(self._video_dir, exist_ok=True)
await asyncio.to_thread(self._save_video, output, video_path, fps)
await asyncio.to_thread(self._save_video, video_frames, video_path, fps)
# Clean up temp input image
if os.path.exists(img_path):
os.remove(img_path)
logger.info(f"Job {job_id}: completed")
await self.db.update_status(job_id, "completed")
except Exception as e:
logger.exception(f"Job {job_id}: failed")
await self.db.update_status(job_id, "failed", error=str(e))
@staticmethod
def _save_video(output: any, path: str, fps: float) -> None:
def _save_video(
frames_iter: Iterator[torch.Tensor], path: str, fps: float
) -> None:
import imageio.v3 as iio
if isinstance(output, torch.Tensor):
frames = output.cpu().numpy()
elif isinstance(output, dict) and "video" in output:
v = output["video"]
frames = v.cpu().numpy() if isinstance(v, torch.Tensor) else v
else:
frames = output
if isinstance(frames, torch.Tensor):
frames = frames.cpu().numpy()
frames = next(frames_iter).cpu().numpy()
if frames.ndim == 4:
frames = frames[0]