vllm serve Lna-Lab/Huihui-gemma-4-26B-A4B-it-abliterated-NVFP4 \
    --max-model-len 8192

docker run --gpus '"device=0"' -p 8016:8016 \
    -v /path/to/model:/models/current:ro \
    --shm-size 16gb \
    vllm/vllm-openai:cu130-nightly \
    vllm serve /models/current --port 8016 --max-model-len 8192

from vllm import LLM, SamplingParams

llm = LLM(
    model="Lna-Lab/Huihui-gemma-4-26B-A4B-it-abliterated-NVFP4",
    max_model_len=8192,
    gpu_memory_utilization=0.85,
)

output = llm.generate(
    ["Explain quantum entanglement in simple terms."],
    SamplingParams(max_tokens=256, temperature=0.7),
)
print(output[0].outputs[0].text)

default_stage:
  default_modifiers:
    QuantizationModifier:
      targets: [Linear]
      ignore: [lm_head, 're:.*embed.*', 're:.*router', 're:.*vision_tower.*']
      scheme: NVFP4

from datasets import load_dataset
from transformers import AutoProcessor, Gemma4ForConditionalGeneration
from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier

model = Gemma4ForConditionalGeneration.from_pretrained(
    "huihui-ai/Huihui-gemma-4-26B-A4B-it-abliterated", dtype="auto"
)
processor = AutoProcessor.from_pretrained(
    "huihui-ai/Huihui-gemma-4-26B-A4B-it-abliterated"
)

recipe = QuantizationModifier(
    targets="Linear",
    scheme="NVFP4",
    ignore=["lm_head", "re:.*embed.*", "re:.*router", "re:.*vision_tower.*"],
)

ds = load_dataset("neuralmagic/calibration", name="LLM", split="train[:20]")

def preprocess_function(example):
    messages = [
        {"role": m["role"], "content": [{"type": "text", "text": m["content"]}]}
        for m in example["messages"]
    ]
    return processor.apply_chat_template(
        messages, return_tensors="pt", padding=False, truncation=True,
        max_length=8192, tokenize=True, add_special_tokens=False,
        return_dict=True, add_generation_prompt=False,
    )

ds = ds.map(preprocess_function, batched=False, remove_columns=ds.column_names)

import torch
def data_collator(batch):
    assert len(batch) == 1
    return {
        key: (torch.tensor(value) if key != "pixel_values"
              else torch.tensor(value, dtype=torch.bfloat16).squeeze(0))
        for key, value in batch[0].items()
    }

oneshot(
    model=model, recipe=recipe, dataset=ds,
    max_seq_length=8192, num_calibration_samples=20,
    data_collator=data_collator,
)

model.save_pretrained("output-NVFP4", save_compressed=True)
processor.save_pretrained("output-NVFP4")