Create app.py

app.py

@@ -0,0 +1,43 @@
+import spaces
+import gradio as gr
+
+from unsloth import FastLanguageModel
+
+max_seq_length = 2048
+dtype = (
+    None
+)
+load_in_4bit = True
+
+model, tokenizer = FastLanguageModel.from_pretrained(
+    model_name="ua-l/gemma-2-9b-legal-uk",
+    max_seq_length=max_seq_length,
+    dtype=dtype,
+    load_in_4bit=load_in_4bit,
+)
+
+FastLanguageModel.for_inference(model)
+
+
+@spaces.GPU
+def predict(question):
+    inputs = tokenizer(
+    [f'''### Question:
+    {question}
+    
+    ### Answer:
+    '''], return_tensors = "pt").to("cuda")
+
+    outputs = model.generate(**inputs, max_new_tokens = 128, use_cache = True)
+    
+    results = tokenizer.batch_decode(outputs, skip_special_tokens=True)
+
+    return results[0]
+
+inputs = gr.Textbox(lines=2, label="Enter a question", value="Як отримати виплати ВПО?")
+
+outputs = gr.JSON(label="Answer")
+
+demo = gr.Interface(fn=predict, inputs=inputs, outputs=outputs)
+demo.launch()
+