with

app.py

@@ -1,17 +1,22 @@
+import os
+from huggingface_hub import login
 import torch
 from transformers import AutoTokenizer, AutoModelForCausalLM
 import gradio as gr
 
+# Авторизация через токен
+hf_token = os.environ.get("HF_TOKEN")
+if hf_token:
+    login(token=hf_token)
+
 model_path = "inceptionai/Llama-3.1-Sherkala-8B-Chat"
-tokenizer = AutoTokenizer.from_pretrained(model_path)
+tokenizer = AutoTokenizer.from_pretrained(model_path, token=hf_token)
 model = AutoModelForCausalLM.from_pretrained(
-    model_path, torch_dtype=torch.bfloat16, device_map="auto"
+    model_path, torch_dtype=torch.bfloat16, device_map="auto", token=hf_token
 )
 
 tokenizer.chat_template = "{% set loop_messages = messages %}{% for message in loop_messages %}{% set content = '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' + message['content'] | trim + '<|eot_id|>' %}{% if loop.index0 == 0 %}{% set content = bos_token + content %}{% endif %}{{ content }}{% endfor %}{% if add_generation_prompt %}{{ '<|start_header_id|>assistant<|end_header_id|>\n\n' }}{% endif %}"
 
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
 def chat_fn(user_input):
     conversation = [{"role": "user", "content": user_input}]
     input_ids = tokenizer.apply_chat_template(
@@ -19,7 +24,7 @@ def chat_fn(user_input):
         tokenize=True,
         add_generation_prompt=True,
         return_tensors="pt"
-    ).to(device)
+    ).to(model.device)
 
     output_ids = model.generate(
         input_ids,
@@ -35,6 +40,5 @@ gr.Interface(
     inputs="text",
     outputs="text",
     title="Sherkala-8B Chat",
-    description="Multilingual LLaMA-3.1 based model (Kazakh, Russian, English)",
-    theme="default"
+    description="Kazakh-Russian-English multilingual chat model",
 ).launch()