dflow_2b / dflow /vae.py
data-archetype's picture
dflow_2b 0.2
6daffc2
Raw History Blame Contribute Delete
7 kB
"""The dinac3_96 autoencoder that maps between images and dflow latents.
dflow_2b generates whitened dinac3_96 latents: 96 channels at 1/16 of the
image resolution. The autoencoder is a separate repository; it is
downloaded at a pinned commit and its own Python package (``dinac3/`` in that
repository) is imported from the download under a private module name, so it
never clashes with an installed copy.
"""
from __future__ import annotations
import importlib.util
import sys
import threading
from pathlib import Path
from types import ModuleType
from typing import Protocol, cast
import torch
from huggingface_hub import snapshot_download
from torch import Tensor
VAE_REPOSITORY = "data-archetype/dinac3_96"
# dinac3_96 v1.0 with its final documentation commit.
VAE_REVISION = "d0ca36daa3e429a510ba465a5f7f8139c66d2f82"
LATENT_CHANNELS = 96
LATENT_STRIDE = 16
_PACKAGE_DIR = "dinac3"
_PRIVATE_MODULE = "_dflow_dinac3"
_DOWNLOAD_PATTERNS = ("config.json", "model.safetensors", f"{_PACKAGE_DIR}/*.py")
_IMPORT_LOCK = threading.Lock()
class _Dinac3Model(Protocol):
"""The part of ``dinac3.Dinac3`` dflow uses."""
def encode(self, images: Tensor) -> Tensor:
"""Whitened float32 latents of ``[-1, 1]`` images."""
...
def decode(self, latents: Tensor, height: int, width: int) -> Tensor:
"""Float32 RGB (unclamped, about ``[-1, 1]``) of whitened latents."""
...
class _Dinac3Factory(Protocol):
"""``dinac3.Dinac3.from_pretrained``."""
def from_pretrained(
self,
path_or_repo: str | Path,
*,
device: torch.device | str,
dtype: torch.dtype,
compile_encoder: bool,
compile_decoder: bool,
revision: str | None,
) -> _Dinac3Model:
"""Load and prepare the autoencoder."""
...
class DFlowVae:
"""Encode images to and decode them from dflow latents.
Images are ``[B, 3, H, W]`` float32 RGB in ``[-1, 1]`` with H and W
multiples of 16; latents are ``[B, 96, H/16, W/16]`` float32, whitened.
"""
def __init__(self, model: _Dinac3Model, *, revision: str) -> None:
"""Wrap a loaded autoencoder; ``revision`` records the commit it came from."""
self.model = model
self.revision = revision
@classmethod
def from_pretrained(
cls,
*,
device: torch.device,
compile: bool,
local_dir: Path | None = None,
revision: str = VAE_REVISION,
cache_dir: Path | None = None,
) -> DFlowVae:
"""Download (or reuse) the pinned dinac3_96 release and load it in bf16.
Args:
device: CUDA device.
compile: ``torch.compile`` the encoder and decoder (dynamic shapes).
local_dir: An existing local copy of the repository to load from
instead of the Hub (offline use).
revision: Hub commit to download; the pinned release by default.
cache_dir: Hugging Face cache directory, or None for the default.
Raises:
FileNotFoundError: If ``local_dir`` lacks the package or weights.
ValueError: On a non-CUDA device.
"""
if device.type != "cuda":
raise ValueError(f"dflow inference needs a CUDA device, got {device}.")
root = (
local_dir
if local_dir is not None
else Path(
snapshot_download(
VAE_REPOSITORY,
revision=revision,
allow_patterns=list(_DOWNLOAD_PATTERNS),
cache_dir=None if cache_dir is None else str(cache_dir),
)
)
)
factory = _import_factory(root)
model = factory.from_pretrained(
root,
device=device,
dtype=torch.bfloat16,
compile_encoder=compile,
compile_decoder=compile,
revision=None,
)
return cls(model, revision=revision)
def decode(self, latents: Tensor) -> Tensor:
"""Decode whitened latents to float32 RGB ``[B, 3, 16h, 16w]``, unclamped."""
height = int(latents.shape[-2]) * LATENT_STRIDE
width = int(latents.shape[-1]) * LATENT_STRIDE
return self.model.decode(latents.float(), height, width)
def encode(self, images: Tensor) -> Tensor:
"""Encode ``[-1, 1]`` RGB images to whitened float32 latents.
Raises:
ValueError: If the image height or width is not a multiple of 16.
"""
height, width = int(images.shape[-2]), int(images.shape[-1])
if height % LATENT_STRIDE or width % LATENT_STRIDE:
raise ValueError(
f"Image size {width}x{height} must be a multiple of {LATENT_STRIDE} "
"on both sides."
)
return self.model.encode(images.float())
def _import_factory(root: Path) -> _Dinac3Factory:
"""Import ``root/dinac3`` under a private module name and return ``Dinac3``.
Raises:
FileNotFoundError: If the package or weights are missing under ``root``.
"""
package_init = root / _PACKAGE_DIR / "__init__.py"
missing = [
path
for path in (package_init, root / "config.json", root / "model.safetensors")
if not path.is_file()
]
if missing:
raise FileNotFoundError(
f"Incomplete {VAE_REPOSITORY} download at {root}: missing {missing}."
)
with _IMPORT_LOCK:
module = sys.modules.get(_PRIVATE_MODULE)
if module is None or Path(module.__file__ or "") != package_init:
module = _load_package(package_init)
return cast("_Dinac3Factory", module.Dinac3)
def _load_package(package_init: Path) -> ModuleType:
"""Execute a package from its ``__init__.py`` under the private name,
replacing any earlier copy and its submodules.
Raises:
ImportError: If Python cannot build a module spec for the file, or the
package fails to import (a missing dependency). The partly
imported modules are removed first, so a retry imports afresh.
"""
spec = importlib.util.spec_from_file_location(
_PRIVATE_MODULE,
package_init,
submodule_search_locations=[str(package_init.parent)],
)
if spec is None or spec.loader is None:
raise ImportError(f"Cannot import the autoencoder package at {package_init}.")
_drop_private_modules()
module = importlib.util.module_from_spec(spec)
sys.modules[_PRIVATE_MODULE] = module
try:
spec.loader.exec_module(module)
except ImportError:
_drop_private_modules()
raise
return module
def _drop_private_modules() -> None:
"""Remove the private autoencoder package and its submodules from
``sys.modules``."""
for name in [
name
for name in sys.modules
if name == _PRIVATE_MODULE or name.startswith(f"{_PRIVATE_MODULE}.")
]:
del sys.modules[name]