Upload 4 files

app.py

@@ -63,7 +63,10 @@ def preprocess_conversations(examples, tokenizer):
                 text += f"<|assistant|>\n{content}\n"
         texts.append(text)
     
-    return tokenizer(texts, truncation=True, max_length=2048, padding=False)
+    # Return tokenized data with labels for language modeling
+    tokenized = tokenizer(texts, truncation=True, max_length=2048, padding=False)
+    tokenized["labels"] = tokenized["input_ids"].copy()
+    return tokenized
 
 # Persistent storage paths
 CHECKPOINT_DIR = Path("/data/checkpoints") if Path("/data").exists() else Path("./checkpoints")
@@ -171,10 +174,11 @@ def train_model(epochs, batch_size, learning_rate, resume=False, progress=gr.Pro
             greater_is_better=False,
         )
         
-        # Data collator
+        # Data collator with padding
         data_collator = DataCollatorForLanguageModeling(
             tokenizer=tokenizer,
-            mlm=False
+            mlm=False,
+            pad_to_multiple_of=8  # Pad to multiple of 8 for efficiency
         )
         
         # Initialize trainer with custom loss