app.py

# 1. Import dependencies
import gradio as gr
import torch
# import spaces # for GPU usage
from PIL import Image, ImageDraw, ImageFont

from transformers import AutoImageProcessor, AutoModelForObjectDetection

# 2. Setup preprocessing and model functions
model_save_path = "Sairii/rt_detrv2_finetuned_trashify_box_detector_v1"

image_processor = AutoImageProcessor.from_pretrained(model_save_path)
image_processor.size = {"height":640,
                        "width":640}

model = AutoModelForObjectDetection.from_pretrained(model_save_path)

# Setup the target device (use GPU if it's accesible)
# Note if you want to use a GPU in your Space, you can use a ZeroGPU: https://huggingface.co/docs/hub/spaces-zerogpu
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)


# Get the id2label dictionary from the model
id2label = model.config.id2label

# Setup a color dictionary
color_dict = {
    "bin": "green",
    "trash": "blue",
    "hand": "purple",
    "trash_arm": "yellow",
    "not_trash": "red",
    "not_bin": "red",
    "not_hand": "red"
}


# 3. Create a function to predict on image
def predict_on_image(image, conf_threshold):
  model.eval()

  # Make a prediction on target image
  with torch.no_grad():
    inputs = image_processor(images = [image], return_tensors = "pt")
    model_outputs = model(**inputs.to(device))

    # Get original size of image
    # PIL.Image.size = width, height
    # But post_process_object_detection requires height, width
    target_sizes = torch.tensor([[image.size[1], image.size[0]]]) # -> [batch_size,  height, width]

    # Post process the raw outputs from the model
    results = image_processor.post_process_object_detection(
        model_outputs = model_outputs,
        threshold = conf_threshold,
        target_sizes = target_sizes
    )[0]

    # Return all data items/objects to the CPU if they aren't already there
    for key, value in results.items():
      try:
        results[key] = value.item().cpu() # can't get scalars as .items() so add try/except block
      except:
        results[key] = value.cpu()


    ### 4. Draw the predictions on the target iamge ###
    draw = ImageDraw.Draw(image)

    # Get a font to write on our image
    font = ImageFont.load_default()

    # Get a list of the detect class names
    detected_class_names_text_labels = []

    # Iterate throught the predictions of the model and draw them on the target image
    for box, score, label in zip(results["boxes"], results["scores"], results["labels"]):
        # Create coordinates
        x, y, x2, y2 = tuple(box.tolist())

        # Get label_name
        label_name = id2label[label.item()]
        targ_color = color_dict[label_name]
        detected_class_names_text_labels.append(label_name)

        # Draw the rectangle
        draw.rectangle(xy=(x, y, x2, y2),
                       outline=targ_color,
                       width=3)

        # Create a text string to display
        text_string_to_show = f"{label_name} ({round(score.item(), 3)})"

        # Draw the text on the image
        draw.text(xy=(x, y),
                  text=text_string_to_show,
                  fill="white",
                  font=font)

    # Remove the draw each time
    del draw

    ### 5. Create logic for outputting information message

    # Set up set of target items to discover
    target_items = {"bin", "trash", "hand"}
    detected_items = set(detected_class_names_text_labels)

    # If no items detected or bin, trash, hand not in detected items, return notifications
    if not detected_items & target_items:
      return_string = {
          f"No trash, bin or hand detected at confidence threshold of {conf_threshold}"
          f"Try another image or lowerin the confidence threshold"
      }
      print(return_string)
      return image, return_string

    # If there are items missing, output what's missing for +1 point
    missing_items = target_items - detected_items
    if missing_items:
      return_string (
          f" Detected the following items: {sorted(detected_class_names_text_labels)}"
          f" Missing the following items: {sorted(missing_items)}"
          "In order to get +1 points, all target items must be detected"
      )
      print(return_string)
      return image, return_string

    # Final case, all items are detected
    return_string = f"+1: Found the following items: {sorted(detected_items)}, thank you for cleaning up your local area"
    return image, return_string

### 6. Setup the demo application to take in image, make a prediction with our model, return the image with drawn predicitons ###

# Write description for our demo application
description = """
Help clean up your local area! Upload an image and get +1 if there is all of the following items detected: trash, bin, hand.

Model is a fine-tuned version of [RT-DETRv2](https://huggingface.co/docs/transformers/main/en/model_doc/rt_detr_v2#transformers.RTDetrV2Config) on the [Trashify dataset](https://huggingface.co/datasets/mrdbourke/trashify_manual_labelled_images).

See the full data loading and training code on [learnhuggingface.com](https://www.learnhuggingface.com/notebooks/hugging_face_object_detection_tutorial).

This version is v1 because the first three versions were using a different model and did not perform as well, see the [README](https://huggingface.co/spaces/mrdbourke/trashify_demo_v4/blob/main/README.md) for more.
"""

# Create the Gradio interface
demo = gr.Interface(
    fn = predict_on_image,
    inputs = [
        gr.Image(type="pil", label="Target Input Image"),
        gr.Slider(0, 1, value=0.3, label="Confidence Threshold set higher for more confident boxes")
    ],
    outputs = [
        gr.Image(type="pil", label = "Target Image Output"),
        gr.Text(label="Text output")
    ],
    description = description,
    title = "Trashify Demo V1",
    examples=[
      ["trashify_examples/trashify_example_1.jpeg", 0.3],
      ["trashify_examples/trashify_example_2.jpeg", 0.3],
      ["trashify_examples/trashify_example_3.jpeg", 0.3],
    ],
    cache_examples = True
)

# Laun demo
demo.launch(debug=True)