termux-train

Tiny Models & LoRA Hub

Pre-configured architectures for on-device Transformers, Whisper speech recognition, and low-rank adapters.

1. Tiny Transformer LM (Decoder-Only with RoPE)

from termux_train import Tensor, nn

model = nn.TinyTransformerLM(
    vocab_size=500,
    d_model=64,
    num_heads=4,
    d_ff=128,
    num_layers=2,
    pos_type="rope",   # Rotary Position Embedding
    tie_weights=True   # Ties token embeddings with LM head
)

# Autoregressive generation with incremental KV cache
generated_tokens = model.generate([1, 10, 45], max_new_tokens=30, temperature=0.7)

2. Tiny Whisper LoRA Speech-to-Text (<30KB)

from termux_train import nn, checkpoint

# Inject LoRA into Attention projections
for block in model.blocks:
    block.attn.q_proj = nn.LoRALinear.from_linear(block.attn.q_proj, rank=4, alpha=8.0)
    block.attn.v_proj = nn.LoRALinear.from_linear(block.attn.v_proj, rank=4, alpha=8.0)

# Save lightweight adapter only (<30KB)
checkpoint.save_lora_adapter(model, "whisper_lora.safetensors")

# Merge into base weights for zero-overhead inference
nn.merge_lora_adapters(model)