feat(tools): texture generation through the documented Stability Matrix workflows
Re-read /home/zwitschi/StabilityMatrix/Data/README.md - it now documents 141 GB of models and four verified, core-node image/texture graphs - and wired the ones that matter into the toolchain. comfyui_generate.py gains a 'texture' subcommand that rebuilds the API graph for each documented workflow, because the files in Workflows/ are ComfyUI UI format and /prompt cannot accept them: albedobase AlbedoBase XL v3.1, 30 steps, CFG 6.0, dpmpp_2m/karras, baked VAE krea2 Krea-2 Turbo NVFP4, 8 steps, CFG 1.0, euler/simple, ConditioningZeroOut, texture-fix VAE qwen21 Qwen-Image 2.1 int8, 25 steps, CFG 1.0, euler/simple, TextEncodeQwenImage21, texture-fix VAE Default pair is albedobase + krea2 - both commercially usable - generated from the same prompt and seed, scored by high-frequency detail, and compared before the winner takes --out. Smoke-tested at 512 px: both rendered, albedobase won 18.91 against 17.51, and the resulting limestone albedo is a plausible material sheet. Tools/README.md documents the profiles with their settings, a licence row per profile (Qwen-Image 2.1 is research-only and must not ship), and the two limits the workflows carry: they produce albedo only, and ComfyUI 0.44 has no seamless sampler, so tiling has to be checked by offsetting the image 50%. Five tests cover the graph shapes; the suite is at 108.
This commit is contained in:
zwitschi committed
1 parent
65e8f01586
commit
70a7e19841
3 files changed
+231
-1
No files matched your search
@@ -66,6 +66,89 @@ def score_image(path):
|
||||
return sum(pixels) / max(1, len(pixels))
|
||||
|
||||
|
||||
# The four verified graphs in the Stability Matrix install
|
||||
# (/home/zwitschi/StabilityMatrix/Data/Workflows, documented in Workflows/texture_generation.md).
|
||||
# They are stored in ComfyUI's UI format, which /prompt cannot take, so the equivalent API graphs are
|
||||
# assembled here with exactly the settings those files use.
|
||||
TEXTURE_PROFILES = {
|
||||
"albedobase": dict(
|
||||
label="AlbedoBase XL v3.1 (SDXL) - game albedo, commercial use with credit",
|
||||
checkpoint="albedobaseXL_v31Large.safetensors",
|
||||
steps=30, cfg=6.0, sampler="dpmpp_2m", scheduler="karras",
|
||||
negative="shadows, perspective, vignette, dramatic lighting, blurry, low quality, text, watermark",
|
||||
negative_style="text", clip_type=None, vae=None,
|
||||
),
|
||||
"krea2": dict(
|
||||
label="Krea-2 Turbo NVFP4 - fast stochastic detail (stone, bark, foliage)",
|
||||
unet="krea2_turbo_nvfp4.safetensors",
|
||||
clip="qwen3vl_4b_int8_convrot.safetensors", clip_type="krea2",
|
||||
vae="texture_fix_vae_for_qwen_image_1_and_krea_2_bf16.safetensors",
|
||||
steps=8, cfg=1.0, sampler="euler", scheduler="simple",
|
||||
negative="", negative_style="zero_out",
|
||||
),
|
||||
"qwen21": dict(
|
||||
label="Qwen-Image 2.1 int8 - material sheets with readable labels (research licence)",
|
||||
unet="qwen_image_2.1_int8_convrot.safetensors",
|
||||
clip="qwen3vl_8b_w4a8.safetensors", clip_type="qwen_image",
|
||||
vae="texture_fix_vae_for_qwen_image_2.1_bf16.safetensors",
|
||||
steps=25, cfg=1.0, sampler="euler", scheduler="simple",
|
||||
negative="", negative_style="none",
|
||||
),
|
||||
}
|
||||
DEFAULT_TEXTURE_PROFILES = ("albedobase", "krea2")
|
||||
|
||||
TEXTURE_PROMPT = ("top-down orthographic game texture, flat even lighting, no shadows, no AO, "
|
||||
"no vignette, PBR albedo, material scan, neutral colours")
|
||||
|
||||
|
||||
def texture_workflow(profile: str, prompt: str, width: int, height: int, seed: int) -> dict:
|
||||
"""API graph for one of the documented texture workflows.
|
||||
|
||||
Text-to-image only: the graphs produce a colour/albedo sheet, and derived normal/roughness maps are
|
||||
made downstream. None of them guarantee wrap-around edges - ComfyUI 0.44 core has no seamless sampler -
|
||||
so a tiling check is still required.
|
||||
"""
|
||||
spec = TEXTURE_PROFILES[profile]
|
||||
graph = {}
|
||||
if "checkpoint" in spec:
|
||||
graph["1"] = {"class_type": "CheckpointLoaderSimple",
|
||||
"inputs": {"ckpt_name": spec["checkpoint"]}}
|
||||
model_ref, clip_ref = ["1", 0], ["1", 1]
|
||||
vae_ref = ["1", 2]
|
||||
else:
|
||||
graph["1"] = {"class_type": "UNETLoader",
|
||||
"inputs": {"unet_name": spec["unet"], "weight_dtype": "default"}}
|
||||
graph["2"] = {"class_type": "CLIPLoader",
|
||||
"inputs": {"clip_name": spec["clip"], "type": spec["clip_type"],
|
||||
"device": "default"}}
|
||||
graph["3"] = {"class_type": "VAELoader", "inputs": {"vae_name": spec["vae"]}}
|
||||
model_ref, clip_ref, vae_ref = ["1", 0], ["2", 0], ["3", 0]
|
||||
|
||||
encode = "TextEncodeQwenImage21" if profile == "qwen21" else "CLIPTextEncode"
|
||||
graph["4"] = {"class_type": encode, "inputs": {"text": prompt, "clip": clip_ref}}
|
||||
if spec["negative_style"] == "zero_out":
|
||||
graph["5"] = {"class_type": "ConditioningZeroOut", "inputs": {"conditioning": ["4", 0]}}
|
||||
negative_ref = ["5", 0]
|
||||
elif spec["negative_style"] == "text":
|
||||
graph["5"] = {"class_type": "CLIPTextEncode",
|
||||
"inputs": {"text": spec["negative"], "clip": clip_ref}}
|
||||
negative_ref = ["5", 0]
|
||||
else:
|
||||
negative_ref = ["4", 0]
|
||||
graph["6"] = {"class_type": "EmptyLatentImage",
|
||||
"inputs": {"width": width, "height": height, "batch_size": 1}}
|
||||
graph["7"] = {"class_type": "KSampler",
|
||||
"inputs": {"seed": seed, "steps": spec["steps"], "cfg": spec["cfg"],
|
||||
"sampler_name": spec["sampler"], "scheduler": spec["scheduler"],
|
||||
"denoise": 1.0, "model": model_ref, "positive": ["4", 0],
|
||||
"negative": negative_ref, "latent_image": ["6", 0]}}
|
||||
graph["8"] = {"class_type": "VAEDecode",
|
||||
"inputs": {"samples": ["7", 0], "vae": vae_ref}}
|
||||
graph["9"] = {"class_type": "SaveImage",
|
||||
"inputs": {"images": ["8", 0], "filename_prefix": f"texture_{profile}"}}
|
||||
return graph
|
||||
|
||||
|
||||
def api(path: str) -> dict:
|
||||
with urllib.request.urlopen(f"{HOST}{path}", timeout=30) as response:
|
||||
return json.loads(response.read().decode())
|
||||
@@ -136,6 +219,69 @@ def wait_for_image(prompt_id: str, timeout: float = 300.0, poll: float = 1.5):
|
||||
return None
|
||||
|
||||
|
||||
def run_graph(graph, out_path, meta) -> int:
|
||||
"""Queue one API graph, wait for the image, write it plus a provenance sidecar."""
|
||||
queued = post("/prompt", {"prompt": graph})
|
||||
prompt_id = queued.get("prompt_id")
|
||||
if not prompt_id:
|
||||
print(f"error: ComfyUI rejected the prompt: {queued}", file=sys.stderr)
|
||||
return 1
|
||||
data = wait_for_image(prompt_id, timeout=meta["timeout"], poll=meta["poll"])
|
||||
if data is None:
|
||||
print("error: no image came back (timeout or empty job)", file=sys.stderr)
|
||||
return 1
|
||||
os.makedirs(os.path.dirname(os.path.abspath(out_path)) or ".", exist_ok=True)
|
||||
with open(out_path, "wb") as handle:
|
||||
handle.write(data)
|
||||
meta = dict(meta)
|
||||
meta.pop("timeout", None)
|
||||
meta.pop("poll", None)
|
||||
meta.update({"tool": "Tools/comfyui_generate.py",
|
||||
"generated_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
|
||||
"workflow": graph, "comfyui_prompt_id": prompt_id,
|
||||
"note": "generated locally; record the model licence in "
|
||||
"THIRD_PARTY_NOTICES.md before shipping"})
|
||||
with open(os.path.splitext(out_path)[0] + ".json", "w") as handle:
|
||||
json.dump(meta, handle, indent=2)
|
||||
print(f"wrote {out_path} (+ sidecar)")
|
||||
return 0
|
||||
|
||||
|
||||
def texture(args) -> int:
|
||||
"""Generate a material sheet with the documented texture workflows, two profiles by default."""
|
||||
width, height = (int(v) for v in args.size.lower().split("x"))
|
||||
base_seed = args.seed if args.seed is not None else random.randrange(0, 2**31)
|
||||
prompt = args.prompt or f"{TEXTURE_PROMPT}, {args.material}"
|
||||
profiles = [p for p in (args.profile, args.alt_profile) if p]
|
||||
if not profiles:
|
||||
profiles = list(DEFAULT_TEXTURE_PROFILES)
|
||||
stem, ext = os.path.splitext(args.out)
|
||||
candidates = []
|
||||
for name in profiles:
|
||||
path = f"{stem}__{name}{ext}"
|
||||
graph = texture_workflow(name, prompt, width, height, base_seed)
|
||||
meta = {"prompt": prompt, "profile": name, "size": [width, height], "seed": base_seed,
|
||||
"settings": {k: TEXTURE_PROFILES[name][k] for k in
|
||||
("steps", "cfg", "sampler", "scheduler")},
|
||||
"licence": TEXTURE_PROFILES[name]["label"],
|
||||
"timeout": args.timeout, "poll": args.poll}
|
||||
if run_graph(graph, path, meta) != 0:
|
||||
print(f"profile {name} failed; continuing")
|
||||
continue
|
||||
candidates.append((name, path, score_image(path)))
|
||||
if not candidates:
|
||||
return 1
|
||||
scored = [c for c in candidates if c[2] is not None]
|
||||
winner = max(scored, key=lambda c: c[2]) if scored else candidates[0]
|
||||
for name, path, score in candidates:
|
||||
print(f" {name}: {'n/a' if score is None else round(score, 2)} {path}")
|
||||
os.replace(winner[1], args.out)
|
||||
if os.path.exists(winner[1] + ".json"):
|
||||
os.replace(winner[1] + ".json", args.out + ".json")
|
||||
print(f"chose {winner[0]} -> {args.out}")
|
||||
return 0
|
||||
|
||||
|
||||
def generate(args) -> int:
|
||||
width, height = (int(v) for v in args.size.lower().split("x"))
|
||||
seed = args.seed if args.seed is not None else random.randrange(0, 2**31)
|
||||
@@ -240,8 +386,24 @@ def main(argv=None) -> int:
|
||||
t.add_argument("--out", required=True)
|
||||
t.add_argument("--timeout", type=float, default=300.0)
|
||||
t.add_argument("--poll", type=float, default=1.5)
|
||||
x = sub.add_parser("texture", help="material sheet via the documented texture workflows")
|
||||
x.add_argument("--material", required=True,
|
||||
help="what to draw, e.g. 'weathered grey limestone ashlar'")
|
||||
x.add_argument("--prompt", default="", help="full prompt, overriding the material shorthand")
|
||||
x.add_argument("--profile", default=DEFAULT_TEXTURE_PROFILES[0],
|
||||
choices=sorted(TEXTURE_PROFILES))
|
||||
x.add_argument("--alt-profile", default=DEFAULT_TEXTURE_PROFILES[1],
|
||||
choices=sorted(TEXTURE_PROFILES) + [""],
|
||||
help="second workflow, generated from the same prompt and seed, then compared")
|
||||
x.add_argument("--size", default="1024x1024")
|
||||
x.add_argument("--seed", type=int, default=None)
|
||||
x.add_argument("--out", required=True)
|
||||
x.add_argument("--timeout", type=float, default=300.0)
|
||||
x.add_argument("--poll", type=float, default=1.5)
|
||||
args = ap.parse_args(argv)
|
||||
|
||||
if args.cmd == "texture":
|
||||
return texture(args)
|
||||
if args.cmd == "models":
|
||||
for name in checkpoints():
|
||||
print(name)
|
||||
|
||||
Reference in new issue
Block a user