"""The dinac3_96 autoencoder that maps between images and dflow latents. dflow_2b generates whitened dinac3_96 latents: 96 channels at 1/16 of the image resolution. The autoencoder is a separate repository; it is downloaded at a pinned commit and its own Python package (``dinac3/`` in that repository) is imported from the download under a private module name, so it never clashes with an installed copy. """ from __future__ import annotations import importlib.util import sys import threading from pathlib import Path from types import ModuleType from typing import Protocol, cast import torch from huggingface_hub import snapshot_download from torch import Tensor VAE_REPOSITORY = "data-archetype/dinac3_96" # dinac3_96 v1.0 with its final documentation commit. VAE_REVISION = "d0ca36daa3e429a510ba465a5f7f8139c66d2f82" LATENT_CHANNELS = 96 LATENT_STRIDE = 16 _PACKAGE_DIR = "dinac3" _PRIVATE_MODULE = "_dflow_dinac3" _DOWNLOAD_PATTERNS = ("config.json", "model.safetensors", f"{_PACKAGE_DIR}/*.py") _IMPORT_LOCK = threading.Lock() class _Dinac3Model(Protocol): """The part of ``dinac3.Dinac3`` dflow uses.""" def encode(self, images: Tensor) -> Tensor: """Whitened float32 latents of ``[-1, 1]`` images.""" ... def decode(self, latents: Tensor, height: int, width: int) -> Tensor: """Float32 RGB (unclamped, about ``[-1, 1]``) of whitened latents.""" ... class _Dinac3Factory(Protocol): """``dinac3.Dinac3.from_pretrained``.""" def from_pretrained( self, path_or_repo: str | Path, *, device: torch.device | str, dtype: torch.dtype, compile_encoder: bool, compile_decoder: bool, revision: str | None, ) -> _Dinac3Model: """Load and prepare the autoencoder.""" ... class DFlowVae: """Encode images to and decode them from dflow latents. Images are ``[B, 3, H, W]`` float32 RGB in ``[-1, 1]`` with H and W multiples of 16; latents are ``[B, 96, H/16, W/16]`` float32, whitened. """ def __init__(self, model: _Dinac3Model, *, revision: str) -> None: """Wrap a loaded autoencoder; ``revision`` records the commit it came from.""" self.model = model self.revision = revision @classmethod def from_pretrained( cls, *, device: torch.device, compile: bool, local_dir: Path | None = None, revision: str = VAE_REVISION, cache_dir: Path | None = None, ) -> DFlowVae: """Download (or reuse) the pinned dinac3_96 release and load it in bf16. Args: device: CUDA device. compile: ``torch.compile`` the encoder and decoder (dynamic shapes). local_dir: An existing local copy of the repository to load from instead of the Hub (offline use). revision: Hub commit to download; the pinned release by default. cache_dir: Hugging Face cache directory, or None for the default. Raises: FileNotFoundError: If ``local_dir`` lacks the package or weights. ValueError: On a non-CUDA device. """ if device.type != "cuda": raise ValueError(f"dflow inference needs a CUDA device, got {device}.") root = ( local_dir if local_dir is not None else Path( snapshot_download( VAE_REPOSITORY, revision=revision, allow_patterns=list(_DOWNLOAD_PATTERNS), cache_dir=None if cache_dir is None else str(cache_dir), ) ) ) factory = _import_factory(root) model = factory.from_pretrained( root, device=device, dtype=torch.bfloat16, compile_encoder=compile, compile_decoder=compile, revision=None, ) return cls(model, revision=revision) def decode(self, latents: Tensor) -> Tensor: """Decode whitened latents to float32 RGB ``[B, 3, 16h, 16w]``, unclamped.""" height = int(latents.shape[-2]) * LATENT_STRIDE width = int(latents.shape[-1]) * LATENT_STRIDE return self.model.decode(latents.float(), height, width) def encode(self, images: Tensor) -> Tensor: """Encode ``[-1, 1]`` RGB images to whitened float32 latents. Raises: ValueError: If the image height or width is not a multiple of 16. """ height, width = int(images.shape[-2]), int(images.shape[-1]) if height % LATENT_STRIDE or width % LATENT_STRIDE: raise ValueError( f"Image size {width}x{height} must be a multiple of {LATENT_STRIDE} " "on both sides." ) return self.model.encode(images.float()) def _import_factory(root: Path) -> _Dinac3Factory: """Import ``root/dinac3`` under a private module name and return ``Dinac3``. Raises: FileNotFoundError: If the package or weights are missing under ``root``. """ package_init = root / _PACKAGE_DIR / "__init__.py" missing = [ path for path in (package_init, root / "config.json", root / "model.safetensors") if not path.is_file() ] if missing: raise FileNotFoundError( f"Incomplete {VAE_REPOSITORY} download at {root}: missing {missing}." ) with _IMPORT_LOCK: module = sys.modules.get(_PRIVATE_MODULE) if module is None or Path(module.__file__ or "") != package_init: module = _load_package(package_init) return cast("_Dinac3Factory", module.Dinac3) def _load_package(package_init: Path) -> ModuleType: """Execute a package from its ``__init__.py`` under the private name, replacing any earlier copy and its submodules. Raises: ImportError: If Python cannot build a module spec for the file, or the package fails to import (a missing dependency). The partly imported modules are removed first, so a retry imports afresh. """ spec = importlib.util.spec_from_file_location( _PRIVATE_MODULE, package_init, submodule_search_locations=[str(package_init.parent)], ) if spec is None or spec.loader is None: raise ImportError(f"Cannot import the autoencoder package at {package_init}.") _drop_private_modules() module = importlib.util.module_from_spec(spec) sys.modules[_PRIVATE_MODULE] = module try: spec.loader.exec_module(module) except ImportError: _drop_private_modules() raise return module def _drop_private_modules() -> None: """Remove the private autoencoder package and its submodules from ``sys.modules``.""" for name in [ name for name in sys.modules if name == _PRIVATE_MODULE or name.startswith(f"{_PRIVATE_MODULE}.") ]: del sys.modules[name]