Fix: Add HF_TOKEN authentication to dataset downloads

The download_from_dataset() function now passes the HF_TOKEN
to hf_hub_download() for authentication when accessing private datasets.

This fixes the "Embeddings not loaded!" error on the HuggingFace Space.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

foundation_engine.py

@@ -234,6 +234,7 @@ def download_from_dataset(filename):
     """Download file from HF Dataset if not present locally"""
     from huggingface_hub import hf_hub_download
     import tempfile
+    import os
 
     # Use /tmp for downloads (has write permissions in Docker)
     download_dir = Path("/tmp/foundation_data")
@@ -247,12 +248,17 @@ def download_from_dataset(filename):
 
     try:
         logger.info(f"Downloading {filename} from {DATASET_REPO}...")
+
+        # Get HF_TOKEN from environment or global hf_token variable
+        token = os.environ.get('HF_TOKEN') or hf_token
+
         downloaded_file = hf_hub_download(
             repo_id=DATASET_REPO,
             filename=filename,
             repo_type="dataset",
             local_dir=download_dir,
-            local_dir_use_symlinks=False
+            local_dir_use_symlinks=False,
+            token=token  # Pass authentication token
         )
         logger.info(f"Downloaded {filename}")
         return Path(downloaded_file)