Fix API to match actual ltx-pipelines 1.1.5 interface
- Use DistilledPipeline constructor (not from_config) - Use ImageConditioningInput namedtuple for image conditioning - Use LoraPathStrengthAndSDOps for LoRA config - Use QuantizationPolicy with fp8_cast sd_ops directly - Update spatial_upsampler to required x2-1.1 model - Update gemma_root to google/gemma-3-12b-it-qat-q4_0-unquantized - Handle video output as Iterator[torch.Tensor] from pipeline - Flesh out README with all model download commands, endpoint docs, env vars, frame/resolution tables
This commit is contained in:
+94
-29
@@ -1,18 +1,24 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import uuid
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime, timezone
|
||||
from typing import Optional
|
||||
|
||||
import torch
|
||||
from ltx_core.loader import LoraPathStrengthAndSDOps
|
||||
from ltx_pipelines import DistilledPipeline
|
||||
from ltx_pipelines.utils.args import ImageConditioningInput
|
||||
from PIL import Image
|
||||
|
||||
from app.config import settings
|
||||
from app.database import JobDB, JobRecord, VALID_FRAME_COUNTS
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class LtxService:
|
||||
def __init__(self, db: JobDB) -> None:
|
||||
@@ -26,7 +32,10 @@ class LtxService:
|
||||
async with self._load_lock:
|
||||
if self.pipeline is not None:
|
||||
return
|
||||
logger.info("Loading DistilledPipeline...")
|
||||
lora_config = self._parse_lora_paths(settings.ltx_loras)
|
||||
quantization = self._build_quantization_policy()
|
||||
|
||||
self.pipeline = await asyncio.to_thread(
|
||||
DistilledPipeline,
|
||||
distilled_checkpoint_path=settings.ltx_distilled_checkpoint,
|
||||
@@ -34,21 +43,60 @@ class LtxService:
|
||||
spatial_upsampler_path=settings.ltx_spatial_upsampler,
|
||||
loras=lora_config,
|
||||
device=settings.ltx_device,
|
||||
quantization=settings.ltx_quantization,
|
||||
quantization=quantization,
|
||||
)
|
||||
logger.info("Pipeline loaded successfully")
|
||||
|
||||
@staticmethod
|
||||
def _parse_lora_paths(lora_specs: list[str]) -> list[dict]:
|
||||
def _parse_lora_paths(lora_specs: list[str]) -> list[LoraPathStrengthAndSDOps]:
|
||||
result = []
|
||||
for spec in lora_specs:
|
||||
if ":" in spec:
|
||||
path, scale = spec.rsplit(":", 1)
|
||||
result.append({"path": path.strip(), "scale": float(scale.strip())})
|
||||
result.append(
|
||||
LoraPathStrengthAndSDOps(
|
||||
path=path.strip(), strength=float(scale.strip()), sd_ops=[]
|
||||
)
|
||||
)
|
||||
else:
|
||||
result.append({"path": spec.strip(), "scale": 1.0})
|
||||
result.append(
|
||||
LoraPathStrengthAndSDOps(path=spec.strip(), strength=1.0, sd_ops=[])
|
||||
)
|
||||
return result
|
||||
|
||||
async def submit_job(self, image: Image.Image, request_params: dict) -> JobRecord:
|
||||
@staticmethod
|
||||
def _build_quantization_policy() -> object | None:
|
||||
q = settings.ltx_quantization
|
||||
if not q:
|
||||
return None
|
||||
if q == "fp8_cast":
|
||||
from ltx_core.quantization.fp8_cast import (
|
||||
TRANSFORMER_LINEAR_DOWNCAST_MAP,
|
||||
UPCAST_DURING_INFERENCE,
|
||||
fp8_cast_fuse_rule,
|
||||
)
|
||||
from ltx_core.quantization.policy import QuantizationPolicy
|
||||
|
||||
return QuantizationPolicy(
|
||||
sd_ops=TRANSFORMER_LINEAR_DOWNCAST_MAP,
|
||||
module_ops=(UPCAST_DURING_INFERENCE,),
|
||||
fuse_rule=fp8_cast_fuse_rule,
|
||||
)
|
||||
if q == "fp8_scaled_mm":
|
||||
from ltx_core.quantization.fp8_scaled_mm import (
|
||||
FP8_SCALED_MM_MODULE_OPS,
|
||||
fp8_scaled_mm_fuse_rule,
|
||||
)
|
||||
from ltx_core.quantization.policy import QuantizationPolicy
|
||||
|
||||
return QuantizationPolicy(
|
||||
module_ops=FP8_SCALED_MM_MODULE_OPS, fuse_rule=fp8_scaled_mm_fuse_rule
|
||||
)
|
||||
return None
|
||||
|
||||
async def submit_job(
|
||||
self, image: Image.Image, request_params: dict[str, object]
|
||||
) -> JobRecord:
|
||||
job_id = uuid.uuid4().hex[:12]
|
||||
now = datetime.now(timezone.utc).isoformat()
|
||||
prompt = request_params.get("prompt")
|
||||
@@ -75,7 +123,7 @@ class LtxService:
|
||||
"width": width,
|
||||
"height": height,
|
||||
"num_frames": num_frames,
|
||||
"fps": fps,
|
||||
"fps": float(fps),
|
||||
"seed": seed,
|
||||
},
|
||||
created_at=now,
|
||||
@@ -88,56 +136,73 @@ class LtxService:
|
||||
self,
|
||||
job_id: str,
|
||||
image: Image.Image,
|
||||
params: dict,
|
||||
params: dict[str, object],
|
||||
) -> None:
|
||||
async with self._job_semaphore:
|
||||
try:
|
||||
logger.info(f"Job {job_id}: starting processing")
|
||||
await self.db.update_status(job_id, "processing")
|
||||
await self.load_pipeline()
|
||||
|
||||
prompt = params.get("prompt", "") or ""
|
||||
width = params.get("width", settings.default_width)
|
||||
height = params.get("height", settings.default_height)
|
||||
num_frames = params.get("num_frames", settings.default_frames)
|
||||
fps = params.get("fps", settings.default_fps)
|
||||
seed = params.get("seed") if params.get("seed") is not None else 42
|
||||
prompt: str = params.get("prompt", "") or ""
|
||||
width: int = params.get("width", settings.default_width)
|
||||
height: int = params.get("height", settings.default_height)
|
||||
num_frames: int = params.get("num_frames", settings.default_frames)
|
||||
fps: float = params.get("fps", settings.default_fps)
|
||||
seed: int = int(params.get("seed")) if params.get("seed") is not None else 42
|
||||
|
||||
# Save image to temp file (ImageConditioningInput expects a path)
|
||||
if image.mode != "RGB":
|
||||
image = image.convert("RGB")
|
||||
image = image.resize((width, height), Image.LANCZOS)
|
||||
|
||||
output = await asyncio.to_thread(
|
||||
img_path = os.path.join(
|
||||
settings.video_output_dir, f"{job_id}_input.png"
|
||||
)
|
||||
os.makedirs(settings.video_output_dir, exist_ok=True)
|
||||
image.save(img_path)
|
||||
|
||||
conditioning = [
|
||||
ImageConditioningInput(
|
||||
path=img_path, frame_idx=0, strength=1.0, crf=33
|
||||
)
|
||||
]
|
||||
|
||||
logger.info(f"Job {job_id}: running inference")
|
||||
result = await asyncio.to_thread(
|
||||
self.pipeline,
|
||||
prompt=prompt,
|
||||
images=image,
|
||||
seed=seed,
|
||||
width=width,
|
||||
height=height,
|
||||
num_frames=num_frames,
|
||||
frame_rate=fps,
|
||||
seed=seed,
|
||||
frame_rate=float(fps),
|
||||
images=conditioning,
|
||||
)
|
||||
|
||||
# result is tuple[Iterator[torch.Tensor], Audio]
|
||||
video_frames: Iterator[torch.Tensor] = result[0]
|
||||
video_path = os.path.join(self._video_dir, f"{job_id}.mp4")
|
||||
os.makedirs(self._video_dir, exist_ok=True)
|
||||
await asyncio.to_thread(self._save_video, output, video_path, fps)
|
||||
await asyncio.to_thread(self._save_video, video_frames, video_path, fps)
|
||||
|
||||
# Clean up temp input image
|
||||
if os.path.exists(img_path):
|
||||
os.remove(img_path)
|
||||
|
||||
logger.info(f"Job {job_id}: completed")
|
||||
await self.db.update_status(job_id, "completed")
|
||||
|
||||
except Exception as e:
|
||||
logger.exception(f"Job {job_id}: failed")
|
||||
await self.db.update_status(job_id, "failed", error=str(e))
|
||||
|
||||
@staticmethod
|
||||
def _save_video(output: any, path: str, fps: float) -> None:
|
||||
def _save_video(
|
||||
frames_iter: Iterator[torch.Tensor], path: str, fps: float
|
||||
) -> None:
|
||||
import imageio.v3 as iio
|
||||
|
||||
if isinstance(output, torch.Tensor):
|
||||
frames = output.cpu().numpy()
|
||||
elif isinstance(output, dict) and "video" in output:
|
||||
v = output["video"]
|
||||
frames = v.cpu().numpy() if isinstance(v, torch.Tensor) else v
|
||||
else:
|
||||
frames = output
|
||||
if isinstance(frames, torch.Tensor):
|
||||
frames = frames.cpu().numpy()
|
||||
frames = next(frames_iter).cpu().numpy()
|
||||
|
||||
if frames.ndim == 4:
|
||||
frames = frames[0]
|
||||
|
||||
Reference in New Issue
Block a user