init

Dockerfile

@@ -0,0 +1,57 @@
+FROM ubuntu:20.04
+
+ARG DEBIAN_FRONTEND=noninteractive
+
+# Install apt-getable dependencies
+RUN apt-get update \
+    && apt-get install -y \
+        build-essential \
+        cmake \
+        git \
+        libeigen3-dev \
+        libopencv-dev \
+        libceres-dev \
+        python3-dev \
+        curl \
+        pkg-config \
+        libcairo2-dev \
+        software-properties-common \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+# Mapmachine requirements
+RUN add-apt-repository ppa:ubuntugis/ppa && \
+    apt-get update && \
+    apt-get -y install libgeos-dev
+
+RUN add-apt-repository ppa:deadsnakes/ppa && \
+    apt-get update && \
+    apt install -y python3.9-dev && \
+    curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
+    python3.9 get-pip.py
+
+ARG REINSTALL_MAPMACHINE=1
+RUN pip3.9 install git+https://github.com/tonyzzzzzz/map-machine
+
+WORKDIR /home/
+
+# OrienterNet Requirements TODO: Install directly from our requirements once our repo is public
+
+RUN git clone https://github.com/mapillary/OpenSfM.git && cd OpenSfM && \
+    pip3.9 install -r requirements.txt 
+
+RUN git clone https://github.com/facebookresearch/OrienterNet.git && cd OrienterNet && \
+    pip3 install -r requirements/full.txt
+
+# MapPerceptionNet extra requirements
+RUN pip3.9 install geojson shapely geopandas mercantile turfpy vt2geojson folium fastapi\
+    geopy gradio pyarrow cloudpickle==2.0.0 urllib3~=1.25.6 scikit-image filelock hydra-core
+
+ARG CACHE_RESET=1
+RUN useradd -m -u 1000 user
+USER user
+WORKDIR /app
+
+RUN git clone https://github.com/MapItAnywhere/MapItAnywhere.git
+COPY --chown=user . /app
+CMD ["python3.9", "-m", "main"]

README.md

@@ -1,10 +1,11 @@
 ---
-title: DataEngine
-emoji: 🌖
-colorFrom: indigo
-colorTo: red
+title: MIA Data Engine
+emoji: 🗺️
+colorFrom: yellow
+colorTo: pink
 sdk: docker
 pinned: false
+app_port: 7860
 ---
 
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference

main.py

@@ -0,0 +1,276 @@
+import requests
+import os
+import sys
+from pathlib import Path
+
+import gradio as gr
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+import geopandas as gpd
+from pyproj.transformer import Transformer
+
+sys.path.append(os.path.dirname(os.path.realpath(__file__)))
+from MapItAnywhere.mia.bev import get_bev
+from MapItAnywhere.mia.fpv import get_fpv
+from MapItAnywhere.mia.fpv import filters
+from MapItAnywhere.mia import logger
+
+def get_city_boundary(query, fetch_shape=False):
+    # Use Nominatim API to get the boundary of the city
+    base_url = "https://nominatim.openstreetmap.org/search"
+    params = {
+        'q': query,
+        'format': 'json',
+        'limit': 1,
+        'polygon_geojson': 1 if fetch_shape else 0
+    }
+
+    headers = {
+        'User-Agent': f'mapperceptionnet_{query}'
+    }
+    response = requests.get(base_url, params=params, headers=headers)
+
+    if response.status_code != 200:
+        logger.error(f"Nominatim error when fetching boundary data for {query}.\n"
+                     f"Status code: {response.status_code}. Content: {response.content}")
+        return None
+    
+    data = response.json()
+
+    if data is None:
+        logger.warn(f"No data returned by Nominatim for {query}")
+        return None
+    
+    # Extract bbox data from the API response
+    bbox_data = data[0]['boundingbox']
+    bbox = {
+        'west': float(bbox_data[2]),
+        'south': float(bbox_data[0]),
+        'east': float(bbox_data[3]),
+        'north': float(bbox_data[1])
+    }
+
+    if fetch_shape:
+        # Extract GeoJSON boundary data from the API response
+        boundary_geojson = data[0]['geojson']
+        boundary_geojson = {
+        "type": "FeatureCollection",
+        "features": [
+            {"type": "Feature",
+            "properties": {},
+            "geometry": boundary_geojson}]
+        }
+        return bbox, boundary_geojson
+    else:
+        return bbox
+    
+def split_dataframe(df, chunk_size = 100): 
+    chunks = list()
+    num_chunks = len(df) // chunk_size + 1
+    for i in range(num_chunks):
+        chunks.append(df[i*chunk_size:(i+1)*chunk_size])
+    return chunks
+
+async def fetch(location, filter_undistort, disable_cam_filter, map_length, mpp):
+    N=1
+    TOTAL_LOOKED_INTO_LIMIT = 10000
+
+    ################ FPV
+    downloader = get_fpv.MapillaryDownloader(os.getenv("MLY_TOKEN"))
+    bbox = get_city_boundary(query=location)
+    tiles = get_fpv.get_tiles_from_boundary(boundary_info=dict(bound_type="auto_bbox", bbox=bbox), zoom=14)
+    np.random.shuffle(tiles)
+    total_looked_into = 0
+    dfs_meta = list()
+    for tile in tiles:
+        image_points_response = await downloader.get_tiles_image_points([tile])
+        if image_points_response is None:
+            continue
+        try:
+            df = get_fpv.parse_image_points_json_data(image_points_response)
+            if len(df) == 0:
+                continue
+            total_looked_into += len(df)
+            df_split = split_dataframe(df, chunk_size=100)
+
+            for df in df_split:
+                image_ids = df["id"]
+                image_infos, num_fail = await get_fpv.fetch_image_infos(image_ids, downloader, infos_dir)
+                df_meta = get_fpv.geojson_feature_list_to_pandas(image_infos.values())
+
+                # Some standardization of the data
+                df_meta["model"] = df_meta["model"].str.lower().str.replace(' ', '').str.replace('_', '')
+                df_meta["make"] = df_meta["make"].str.lower().str.replace(' ', '').str.replace('_', '')
+
+                if filter_undistort:
+                    fp = no_cam_filter_pipeline if disable_cam_filter else filter_pipeline
+                    df_meta = fp(df_meta)
+                
+                dfs_meta.append(df_meta)
+                total_rows = sum([len(x) for x in dfs_meta])
+                if total_rows > N:
+                    break
+                elif total_looked_into > TOTAL_LOOKED_INTO_LIMIT:
+                    yield (f"Went through {total_looked_into} images and could not find images satisfying the filters."
+                           "\nPlease rerun or run the data engine locally for bulk time consuming operations.", None, None)
+                    return
+            if total_rows > N:
+                break
+        except:
+            pass
+
+    df_meta = pd.concat(dfs_meta)
+    df_meta = df_meta.sample(N)
+
+    # Calc derrivative attributes
+    df_meta["loc_descrip"] = filters.haversine_np(
+        lon1=df_meta["geometry.long"], lat1=df_meta["geometry.lat"],
+        lon2=df_meta["computed_geometry.long"], lat2=df_meta["computed_geometry.lat"]
+    )
+
+    df_meta["angle_descrip"] = filters.angle_dist(
+        df_meta["compass_angle"],
+        df_meta["computed_compass_angle"]
+    )
+
+    for index, row in df_meta.iterrows():
+        desc = list()
+        # Display attributes
+        keys = ["id", "geometry.long", "geometry.lat", "compass_angle",
+                "loc_descrip", "angle_descrip",
+                "make", "model", "camera_type", 
+                "quality_score"]
+        for k in keys:
+            v = row[k]
+            if isinstance(v, float):
+                v = f"{v:.4f}"
+            bullet = f"{k}: {v}"
+            desc.append(bullet)
+        metadata_fmt = "\n".join(desc)
+    
+    yield metadata_fmt, None, None
+    image_urls = list(df_meta.set_index("id")["thumb_2048_url"].items())
+    num_fail = await get_fpv.fetch_images_pixels(image_urls, downloader, raw_image_dir)
+    if num_fail > 0:
+        logger.error(f"Failed to download {num_fail} images.")
+    
+    seq_to_image_ids = df_meta.groupby('sequence')['id'].agg(list).to_dict()
+    lon_center = (bbox['east'] + bbox['west']) / 2
+    lat_center = (bbox['north'] + bbox['south']) / 2
+    projection = get_fpv.Projection(lat_center, lon_center, max_extent=200e3)
+
+    df_meta.index = df_meta["id"]
+    image_infos = df_meta.to_dict(orient="index")
+    process_sequence_args = get_fpv.default_cfg
+
+    if filter_undistort:
+        for seq_id, seq_image_ids in seq_to_image_ids.items():
+            try: 
+                d, pi = get_fpv.process_sequence(
+                    seq_image_ids,
+                    image_infos,
+                    projection,
+                    process_sequence_args,
+                    raw_image_dir,
+                    out_image_dir,
+                )
+                if d is None or pi is None:
+                    raise Exception("process_sequence returned None")
+            except Exception as e:
+                logger.error(f"Failed to process sequence {seq_id} skipping it. Error: {repr(e)}.")
+        
+        fpv = plt.imread(out_image_dir/ f"{row['id']}_undistorted.jpg")
+    else:
+        fpv = plt.imread(raw_image_dir/ f"{row['id']}.jpg")
+    yield metadata_fmt, fpv, None
+    ################ BEV
+    df = df_meta
+    # convert pandas dataframe to geopandas dataframe
+    gdf = gpd.GeoDataFrame(df, 
+                        geometry=gpd.points_from_xy(
+                            df['computed_geometry.long'], 
+                            df['computed_geometry.lat']), 
+                            crs=4326)
+        
+    # convert the geopandas dataframe to UTM
+    utm_crs = gdf.estimate_utm_crs()
+    gdf_utm = gdf.to_crs(utm_crs)
+    transformer = Transformer.from_crs(utm_crs, 4326)
+        # load OSM data, if available
+    padding = 50
+    # calculate the required distance from the center to the edge of the image 
+    # so that the image will not be out of bounds when we rotate it
+    map_length = map_length
+    map_length = np.ceil(np.sqrt(map_length**2 + map_length**2))
+    distance = map_length * mpp
+
+    # create bounding boxes for each point
+    gdf_utm['bounding_box_utm_p1'] = gdf_utm.apply(lambda row: (
+        row.geometry.x - distance - padding,
+        row.geometry.y - distance - padding,
+    ), axis=1)
+
+    gdf_utm['bounding_box_utm_p2'] = gdf_utm.apply(lambda row: (
+        row.geometry.x + distance + padding,
+        row.geometry.y + distance + padding,
+    ), axis=1)
+
+    # convert the bounding box back to lat, long
+    gdf_utm['bounding_box_lat_long_p1'] = gdf_utm.apply(lambda row: transformer.transform(*row['bounding_box_utm_p1']), axis=1)
+    gdf_utm['bounding_box_lat_long_p2'] = gdf_utm.apply(lambda row: transformer.transform(*row['bounding_box_utm_p2']), axis=1)
+    gdf_utm['bbox_min_lat'] = gdf_utm['bounding_box_lat_long_p1'].apply(lambda x: x[0])
+    gdf_utm['bbox_min_long'] = gdf_utm['bounding_box_lat_long_p1'].apply(lambda x: x[1])
+    gdf_utm['bbox_max_lat'] = gdf_utm['bounding_box_lat_long_p2'].apply(lambda x: x[0])
+    gdf_utm['bbox_max_long'] = gdf_utm['bounding_box_lat_long_p2'].apply(lambda x: x[1])
+    gdf_utm['bbox_formatted'] = gdf_utm.apply(lambda row: f"{row['bbox_min_long']},{row['bbox_min_lat']},{row['bbox_max_long']},{row['bbox_max_lat']}", axis=1)
+
+    # iterate over the dataframe and get BEV images
+    jobs = gdf_utm[['id', 'bbox_formatted', 'computed_compass_angle']] # only need the id and bbox_formatted columns for the jobs
+    jobs = jobs.to_dict(orient='records').copy()
+        
+    get_bev.get_bev_from_bbox_worker_init(osm_cache_dir, bev_dir, semantic_mask_dir, rendered_mask_dir, 
+                                          "MapItAnywhere/mia/bev/styles/mia.yml", map_length, mpp, 
+                                          None, True, False, True, True, 1)
+    for job_dict in jobs:
+        get_bev.get_bev_from_bbox_worker(job_dict)
+    
+    bev = plt.imread(rendered_mask_dir / f"{row['id']}.png")
+
+    yield metadata_fmt, fpv, bev
+
+filter_pipeline = filters.FilterPipeline.load_from_yaml("MapItAnywhere/mia/fpv/filter_pipelines/mia.yaml")
+filter_pipeline.verbose=False
+no_cam_filter_pipeline = filters.FilterPipeline.load_from_yaml("MapItAnywhere/mia/fpv/filter_pipelines/mia_rural.yaml")
+no_cam_filter_pipeline.verbose=False
+
+loc = Path(".")
+infos_dir =loc / "infos_dir"
+raw_image_dir = loc / "raw_images"
+out_image_dir = loc / "images"
+osm_cache_dir = loc / "osm_cache"
+bev_dir = loc / "bev_raw"
+semantic_mask_dir = loc / "semantic_masks" 
+rendered_mask_dir = loc / "rendered_semantic_masks" 
+
+all_dirs = [loc, osm_cache_dir, bev_dir, semantic_mask_dir, rendered_mask_dir, out_image_dir, raw_image_dir]
+for d in all_dirs:
+    os.makedirs(d, exist_ok=True)
+
+logger.info(f"Current working directory: {os.getcwd()}, listdir: {os.listdir('.')}")
+
+demo = gr.Interface(
+    fn=fetch,
+    inputs=[gr.Text("Pittsburgh, PA, United States", label="Location"),
+            gr.Checkbox(value=False, label="Filter & Undistort"),
+            gr.Checkbox(value=False, label="Disable camera model filtering"),
+            gr.Slider(minimum=64, maximum=512, step=1, label="BEV Dimension", value=224),
+            gr.Slider(minimum=0.1, maximum=2, label="Meters Per Pixel", value=0.5)],
+    outputs=[gr.Text(label="METADATA"), gr.Image(label="FPV"), gr.Image(label="BEV")],
+    title="MapItAnywhere (Data Engine)",
+    description="A demo showcasing samples of MIA's capability to retrieve FPV-BEV pairs worldwide."
+                "For bulk download/heavy filtering please visit the github and follow the instructions to run locally"
+)
+
+logger.info("Starting server")
+demo.launch(server_name="0.0.0.0", server_port=7860,share=False)