add code

app.py

@@ -0,0 +1,503 @@
+#%%
+import xarray as xr
+from siphon.catalog import TDSCatalog
+import numpy as np
+import matplotlib.pyplot as plt
+import pandas as pd
+import matplotlib.colors as mcolors
+import streamlit as st
+import datetime
+import matplotlib.dates as mdates
+from scipy.interpolate import griddata
+import folium
+import branca.colormap as cm
+
+@st.cache_data(ttl=60)
+def find_latest_meps_file():
+    # The MEPS dataset: https://github.com/metno/NWPdocs/wiki/MEPS-dataset
+    today = datetime.datetime.today()
+    catalog_url = f"https://thredds.met.no/thredds/catalog/meps25epsarchive/{today.year}/{today.month:02d}/{today.day:02d}/catalog.xml"
+    file_url_base = f"https://thredds.met.no/thredds/dodsC/meps25epsarchive/{today.year}/{today.month:02d}/{today.day:02d}"
+    # Get the datasets from the catalog
+    catalog = TDSCatalog(catalog_url)
+    datasets = [s for s in catalog.datasets if "meps_det_ml" in s]
+    file_path = f"{file_url_base}/{sorted(datasets)[-1]}"
+    return file_path
+
+
+@st.cache_data()
+def load_meps_for_location(file_path=None, altitude_min=0, altitude_max=3000):
+    """
+    file_path=None
+    altitude_min=0
+    altitude_max=3000
+    """
+
+    if file_path is None:
+        file_path = find_latest_meps_file()
+
+    x_range= "[220:1:300]"
+    y_range= "[420:1:500]"
+    time_range = "[0:1:66]"
+    hybrid_range = "[25:1:64]"
+    height_range = "[0:1:0]"
+
+    params = {
+        "x": x_range,
+        "y": y_range,
+        "time": time_range,
+        "hybrid": hybrid_range,
+        "height": height_range,
+        "longitude": f"{y_range}{x_range}",
+        "latitude": f"{y_range}{x_range}",
+        "air_temperature_ml": f"{time_range}{hybrid_range}{y_range}{x_range}",
+        "ap" : f"{hybrid_range}",
+        "b" : f"{hybrid_range}",
+        "surface_air_pressure": f"{time_range}{height_range}{y_range}{x_range}",
+        "x_wind_ml": f"{time_range}{hybrid_range}{y_range}{x_range}",
+        "y_wind_ml": f"{time_range}{hybrid_range}{y_range}{x_range}",
+    }
+
+    path = f"{file_path}?{','.join(f'{k}{v}' for k, v in params.items())}"
+
+    subset = xr.open_dataset(path, cache=True)
+    subset.load()
+
+    #%% get geopotential
+    time_range_sfc = "[0:1:0]"
+    surf_params = {
+        "x": x_range,
+        "y": y_range,
+        "time": f"{time_range}",
+        "surface_geopotential": f"{time_range_sfc}[0:1:0]{y_range}{x_range}",
+        "air_temperature_0m": f"{time_range}[0:1:0]{y_range}{x_range}",
+    } 
+    file_path_surf = f"{file_path.replace('meps_det_ml','meps_det_sfc')}?{','.join(f'{k}{v}' for k, v in surf_params.items())}"
+
+    # Load surface parameters and merge into the main dataset
+    surf = xr.open_dataset(file_path_surf, cache=True)
+    # Convert the surface geopotential to elevation
+    elevation = (surf.surface_geopotential / 9.80665).squeeze()
+    #elevation.plot()
+    subset['elevation'] = elevation
+    air_temperature_0m = surf.air_temperature_0m.squeeze()
+    subset['air_temperature_0m'] = air_temperature_0m
+    # subset.elevation.plot()
+    #%%
+    def hybrid_to_height(ds):
+        """
+        ds = subset
+        """
+        # Constants
+        R = 287.05  # Gas constant for dry air
+        g = 9.80665  # Gravitational acceleration
+
+        # Calculate the pressure at each level
+        p = ds['ap'] + ds['b'] * ds['surface_air_pressure']#.mean("ensemble_member")
+
+        # Get the temperature at each level
+        T = ds['air_temperature_ml']#.mean("ensemble_member")
+
+        # Calculate the height difference between each level and the surface
+        dp = ds['surface_air_pressure'] - p  # Pressure difference
+        dT = T - T.isel(hybrid=-1)  # Temperature difference relative to the surface
+        dT_mean = 0.5 * (T + T.isel(hybrid=-1))  # Mean temperature
+
+        # Calculate the height using the hypsometric equation
+        dz = (R * dT_mean / g) * np.log(ds['surface_air_pressure'] / p)
+
+        return dz
+    
+    
+    altitude = hybrid_to_height(subset).mean("time").squeeze().mean("x").mean("y")
+    subset = subset.assign_coords(altitude=('hybrid', altitude.data))
+    subset = subset.swap_dims({'hybrid': 'altitude'})
+
+    # filter subset on altitude ranges
+    subset = subset.where((subset.altitude >= altitude_min) & (subset.altitude <= altitude_max), drop=True).squeeze()
+
+    wind_speed = np.sqrt(subset['x_wind_ml']**2 + subset['y_wind_ml']**2)
+    subset = subset.assign(wind_speed=(('time', 'altitude','y','x'), wind_speed.data))
+
+    
+    subset['thermal_temp_diff'] = compute_thermal_temp_difference(subset)
+    #subset = subset.assign(thermal_temp_diff=(('time', 'altitude','y','x'), thermal_temp_diff.data))
+
+    # Find the indices where the thermal temperature difference is zero or negative
+    # Create tiny value at ground level to avoid finding the ground as the thermal top
+    thermal_temp_diff = subset['thermal_temp_diff'] 
+    thermal_temp_diff = thermal_temp_diff.where(
+        (thermal_temp_diff.sum("altitude")>0)|(subset['altitude']!=subset.altitude.min()), 
+        thermal_temp_diff + 1e-6)
+    indices = (thermal_temp_diff > 0).argmax(dim="altitude")
+    # Get the altitudes corresponding to these indices
+    thermal_top = subset.altitude[indices]
+    subset = subset.assign(thermal_top=(('time', 'y', 'x'), thermal_top.data))
+    subset = subset.set_coords(["latitude", "longitude"])
+
+    return subset
+
+
+#%%
+def compute_thermal_temp_difference(subset):
+    lapse_rate = 0.0098
+    ground_temp = subset.air_temperature_0m-273.3
+    air_temp = (subset['air_temperature_ml']-273.3)#.ffill(dim='altitude')
+
+    # dimensions
+    # 'air_temperature_ml'  altitude: 4 y: 3, x: 3
+    # 'elevation'                       y: 3  x: 3
+    # 'altitude'            altitude: 4
+
+    # broadcast ground temperature to all altitudes, but let it decrease by lapse rate
+    altitude_diff = subset.altitude - subset.elevation
+    altitude_diff = altitude_diff.where(altitude_diff >= 0, 0)
+    temp_decrease = lapse_rate * altitude_diff
+    ground_parcel_temp = ground_temp - temp_decrease
+    thermal_temp_diff = (ground_parcel_temp - air_temp).clip(min=0)
+    return thermal_temp_diff
+
+def wind_and_temp_colorscales(wind_max=20, tempdiff_max=8):
+    # build colorscale for thermal temperature difference
+    wind_colors =    ["grey",   "blue",   "green",    "yellow",   "red",  "purple"]
+    wind_positions = [0,        0.5,          3,          7,         12,     20]  # transition points
+    wind_positions_norm = [i/wind_max for i in wind_positions]
+
+    # Create the colormap
+    windcolors = mcolors.LinearSegmentedColormap.from_list("", list(zip(wind_positions_norm, wind_colors)))
+
+
+    # build colorscale for thermal temperature difference
+    thermal_colors =        ['white',   'white',    'red',  'violet',   "darkviolet"]
+    thermal_positions =     [0,         0.2,        2.0,    4,          8]
+    thermal_positions_norm = [i/tempdiff_max for i in thermal_positions]
+
+    # Create the colormap
+    tempcolors = mcolors.LinearSegmentedColormap.from_list("", list(zip(thermal_positions_norm, thermal_colors)))
+    return windcolors, tempcolors
+
+@st.cache_data(ttl=60)
+def create_wind_map(_subset,  x_target, y_target, altitude_max=4000, date_start=None, date_end=None):
+    """
+    altitude_max = 3000
+    date_start = None
+    date_end = None
+    """
+    subset = _subset
+
+
+
+    wind_min, wind_max = 0.3, 20
+    tempdiff_min, tempdiff_max = 0, 8
+    windcolors, tempcolors = wind_and_temp_colorscales(wind_max, tempdiff_max)
+    # Filter location
+    windplot_data = subset.sel(x=x_target, y=y_target, method="nearest")
+    
+    # Filter time periods and altitudes
+    if date_start is None:
+        date_start = datetime.datetime.fromtimestamp(subset.time.min().values.astype('int64') / 1e9)
+    if date_end is None:
+        date_end = datetime.datetime.fromtimestamp(subset.time.max().values.astype('int64') / 1e9)
+    new_timestamps = pd.date_range(date_start, date_end, 20)
+    
+    new_altitude = np.arange(windplot_data.elevation.mean(), altitude_max, altitude_max/20)
+    windplot_data = windplot_data.interp(altitude=new_altitude, time=new_timestamps)
+
+    # BUILD PLOT
+    fig, ax = plt.subplots(figsize=(15, 7))
+    contourf = ax.contourf(windplot_data.time, windplot_data.altitude, windplot_data.thermal_temp_diff.T, cmap=tempcolors, alpha=0.5, vmin=0, vmax=8)
+    fig.colorbar(contourf, ax=ax, label='Thermal Temperature Difference (°C)', pad=0.01, orientation='vertical')
+    
+    # Wind quiver plot
+    quiverplot = windplot_data.plot.quiver(
+        x='time', y='altitude', u='x_wind_ml', v='y_wind_ml', 
+        hue="wind_speed", 
+        cmap = windcolors,
+        vmin=wind_min, vmax=wind_max,
+        alpha=0.5,
+        pivot="middle",# headwidth=4, headlength=6,
+        ax=ax  # Add this line to plot on the created axes 
+    )
+    quiverplot.colorbar.set_label("Wind Speed  [m/s]")
+    quiverplot.colorbar.pad = 0.01
+
+    # fill bottom with brown color
+    plt.ylim(bottom=0)
+    ax.fill_between(windplot_data.time, 0, windplot_data.elevation.mean(), color="brown", alpha=0.5)
+
+
+    ax.xaxis.set_major_formatter(mdates.DateFormatter('%H:%M'))
+    # normalize wind speed for color mapping
+    norm = plt.Normalize(wind_min, wind_max)
+
+    # add numerical labels to the plot
+    for x, t in enumerate(windplot_data.time.values):
+        for y, alt in enumerate(windplot_data.altitude.values):
+            color = windcolors(norm(windplot_data.wind_speed[x,y]))
+            ax.text(t+5, alt+20, f"{windplot_data.wind_speed[x,y]:.1f}", size=6, color=color)
+    plt.title(f"Wind and thermals in point starting at {date_start.strftime('%Y-%m-%d')} (UTC)")
+    plt.yscale("linear")
+    return fig
+
+#%%
+@st.cache_data(ttl=7200)
+def create_sounding(_subset, date, hour, x_target, y_target, altitude_max=3000):
+    """
+    date = "2024-05-12"
+    hour = "15"
+    x_target = 5
+    y_target = 5
+    """
+    subset = _subset
+    lapse_rate = 0.0098 # in degrees Celsius per meter
+    subset = subset.where(subset.altitude< altitude_max,drop=True)
+    # Create a figure object
+    fig, ax = plt.subplots()
+
+    # Define the dry adiabatic lapse rate
+    def add_dry_adiabatic_lines(ds):
+        # Define a range of temperatures at sea level
+        T0 = np.arange(-40, 40, 5)  # temperatures from -40°C to 40°C in steps of 10°C
+
+        # Create a 2D grid of temperatures and altitudes
+        T0, altitude = np.meshgrid(T0, ds.altitude)
+
+        # Calculate the temperatures at each altitude
+        T_adiabatic = T0 - lapse_rate * altitude
+
+        # Plot the dry adiabatic lines
+        for i in range(T0.shape[1]):
+            ax.plot(T_adiabatic[:, i], ds.altitude, 'r:', alpha=0.5)
+
+    # Plot the actual temperature profiles
+    time_str = f"{date} {hour}:00:00"
+    # find x and y values cloeset to given latitude and longitude
+
+    ds_time = subset.sel(time=time_str, x=x_target,y=y_target, method="nearest")
+    T = (ds_time['air_temperature_ml'].values-273.3)  # in degrees Celsius
+    ax.plot(T, ds_time.altitude, label=f"temp {pd.to_datetime(time_str).strftime('%H:%M')}")
+
+    # Define the surface temperature
+    T_surface = T[-1]+3
+    T_parcel = T_surface - lapse_rate * ds_time.altitude
+
+    # Plot the temperature of the rising air parcel
+    filter = T_parcel>T
+    ax.plot(T_parcel[filter], ds_time.altitude[filter], label='Rising air parcel',color="green")
+
+    add_dry_adiabatic_lines(ds_time)
+
+    ax.set_xlabel('Temperature (°C)')
+    ax.set_ylabel('Altitude (m)')
+    ax.set_title(f'Temperature Profile and Dry Adiabatic Lapse Rate for {date} {hour}:00')
+    ax.legend(title='Time')
+    xmin, xmax = ds_time['air_temperature_ml'].min().values-273.3, ds_time['air_temperature_ml'].max().values-273.3+3
+    ax.set_xlim(xmin, xmax)
+    ax.grid(True)
+
+    # Return the figure object
+    return fig
+
+@st.cache_data(ttl=7200)
+def build_map_overlays(_subset, date=None, hour=None):
+    """
+    date = "2024-05-13"
+    hour = "15"
+    x_target=None
+    y_target=None
+    """
+    subset = _subset
+    
+    # Get the latitude and longitude values from the dataset
+    latitude_values = subset.latitude.values.flatten()
+    longitude_values = subset.longitude.values.flatten()
+    thermal_top_values = subset.thermal_top.sel(time=f"{date}T{hour}").values.flatten()
+    #thermal_top_values = subset.elevation.mean("altitude").values.flatten()
+    # Convert the irregular grid data into a regular grid
+    step_lon, step_lat = subset.longitude.diff("x").quantile(0.1).values, subset.latitude.diff("y").quantile(0.1).values
+    grid_x, grid_y = np.mgrid[min(latitude_values):max(latitude_values):step_lat, min(longitude_values):max(longitude_values):step_lon]
+    grid_z = griddata((latitude_values, longitude_values), thermal_top_values, (grid_x, grid_y), method='linear')
+    grid_z = np.nan_to_num(grid_z, copy=False, nan=0)
+    # Normalize the grid data to a range suitable for image display
+    heightcolor = cm.LinearColormap(
+        colors = ['white',  'white',     'green',   'yellow', 'orange','red', 'darkblue'], 
+        index  = [0,        500,        1000,       1500,   2000,     2500,       3000], 
+        vmin=0, vmax=3000, 
+        caption='Thermal Height (m)')
+
+
+    bounds = [[min(latitude_values), min(longitude_values)], [max(latitude_values), max(longitude_values)]]
+    img_overlay = folium.raster_layers.ImageOverlay(image=grid_z, bounds=bounds, colormap=heightcolor, opacity=0.4, mercator_project=True, origin="lower",pixelated=False)
+
+    return img_overlay, heightcolor
+
+#%%
+import pyproj
+def latlon_to_xy(lat, lon):
+    crs = pyproj.CRS.from_cf(
+        {
+            "grid_mapping_name": "lambert_conformal_conic",
+            "standard_parallel": [63.3, 63.3],
+            "longitude_of_central_meridian": 15.0,
+            "latitude_of_projection_origin": 63.3,
+            "earth_radius": 6371000.0,
+        }
+    )
+    # Transformer to project from ESPG:4368 (WGS:84) to our lambert_conformal_conic
+    proj = pyproj.Proj.from_crs(4326, crs, always_xy=True)
+
+    # Compute projected coordinates of lat/lon point
+    X,Y = proj.transform(lon,lat)
+    return X,Y
+# %%
+def show_forecast():
+
+    with st.spinner('Fetching data...'):        
+        @st.cache_data
+        def load_data(filepath):
+            local=False
+            if local:
+                subset = xr.open_dataset("subset.nc")
+            else:
+                subset = load_meps_for_location(filepath)
+                subset.to_netcdf("subset.nc")
+            return subset
+        
+        if "file_path" not in st.session_state:
+            st.session_state.file_path = find_latest_meps_file()
+        subset = load_data(st.session_state.file_path)
+
+    def date_controls():
+        
+        start_stop_time = [subset.time.min().values.astype('M8[ms]').astype('O'), subset.time.max().values.astype('M8[ms]').astype('O')]
+        now = datetime.datetime.now().replace(minute=0, second=0, microsecond=0)
+
+        if "forecast_date" not in st.session_state:
+            st.session_state.forecast_date = now.date()
+        if "forecast_time" not in st.session_state:
+            st.session_state.forecast_time = datetime.time(14,0)
+        if "forecast_length" not in st.session_state:
+            st.session_state.forecast_length = 1
+        if "altitude_max" not in st.session_state:
+            st.session_state.altitude_max = 3000
+        if "target_latitude" not in st.session_state:
+            st.session_state.target_latitude = 61.22908
+        if "target_longitude" not in st.session_state:
+            st.session_state.target_longitude = 7.09674
+        col1, col_date, col_time, col3 = st.columns([0.2,0.6,0.2,0.2])
+
+        with col1:
+            if st.button("⏮️", use_container_width=True):
+                st.session_state.forecast_date -= datetime.timedelta(days=1)
+        with col3:
+            if st.button("⏭️", use_container_width=True, disabled=(st.session_state.forecast_date == start_stop_time[1])):
+                st.session_state.forecast_date += datetime.timedelta(days=1)
+        with col_date:
+            st.session_state.forecast_date = st.date_input(
+                "Start date", 
+                value=st.session_state.forecast_date, 
+                min_value=start_stop_time[0], 
+                max_value=start_stop_time[1], 
+                label_visibility="collapsed",
+                disabled=True
+                )
+        with col_time:
+            st.session_state.forecast_time = st.time_input("Start time", value=st.session_state.forecast_time, step=3600,disabled=False,label_visibility="collapsed")
+
+    date_controls()
+    time_start = datetime.time(0, 0)
+    # convert subset.attrs['min_time']='2024-05-11T06:00:00Z' into datetime
+    min_time = datetime.datetime.strptime(subset.attrs['min_time'], "%Y-%m-%dT%H:%M:%SZ")
+    date_start = datetime.datetime.combine(st.session_state.forecast_date, time_start)
+    date_start = max(date_start, min_time)
+    date_end= datetime.datetime.combine(st.session_state.forecast_date+datetime.timedelta(days=st.session_state.forecast_length), datetime.time(0, 0))
+
+    ## MAP
+    with st.expander("Map", expanded=True):
+        from streamlit_folium import st_folium
+        st.cache_data(ttl=30)
+        def build_map(date, hour):
+            m = folium.Map(location=[61.22908, 7.09674], zoom_start=9, tiles="openstreetmap")
+            img_overlay, heightcolor = build_map_overlays(subset, date=date, hour=hour)
+            
+            img_overlay.add_to(m)
+            m.add_child(heightcolor,name="Thermal Height (m)")
+            m.add_child(folium.LatLngPopup())
+            return m
+        m = build_map(date = st.session_state.forecast_date,hour=st.session_state.forecast_time)
+        map=st_folium(m)
+        def get_pos(lat,lng):
+            return lat,lng
+        if map['last_clicked'] is not None:
+            st.session_state.target_latitude, st.session_state.target_longitude = get_pos(map['last_clicked']['lat'],map['last_clicked']['lng'])
+    
+    x_target, y_target = latlon_to_xy(st.session_state.target_latitude, st.session_state.target_longitude)
+    wind_fig = create_wind_map(
+                subset,
+                date_start=date_start, 
+                date_end=date_end, 
+                altitude_max=st.session_state.altitude_max,
+                x_target=x_target,
+                y_target=y_target)
+    st.pyplot(wind_fig)
+    plt.close()
+    
+
+    with st.expander("More settings", expanded=False):
+        st.session_state.forecast_length = st.number_input("multiday", 1, 3, 1, step=1,)
+        st.session_state.altitude_max = st.number_input("Max altitude", 0, 4000, 3000, step=500)
+    
+    ############################
+    ######### SOUNDING #########
+    ############################
+    st.markdown("---")
+    with st.expander("Sounding", expanded=False):
+        date = datetime.datetime.combine(st.session_state.forecast_date, st.session_state.forecast_time)
+
+        with st.spinner('Building sounding...'):
+            sounding_fig = create_sounding(
+                subset, 
+                date=date.date(), 
+                hour=date.hour, 
+                altitude_max=st.session_state.altitude_max,
+                x_target=x_target,
+                y_target=y_target)
+        st.pyplot(sounding_fig)
+        plt.close()
+
+    st.markdown("Wind and sounding data from MEPS model (main model used by met.no), including the estimated ground temperature. Ive probably made many errors in this process.")
+
+    # Download new forecast if available
+    st.session_state.file_path = find_latest_meps_file()
+    subset = load_data(st.session_state.file_path)
+
+
+if __name__ == "__main__":
+    run_streamlit = True
+    if run_streamlit:
+        st.set_page_config(page_title="PGWeather",page_icon="🪂", layout="wide")
+        show_forecast()
+    else:
+        lat = 61.22908
+        lon = 7.09674
+        x_target, y_target = latlon_to_xy(lat, lon)
+        
+        dataset_file_path = find_latest_meps_file()
+        local=True
+        if local:
+            subset = xr.open_dataset("subset.nc")
+        else:
+            subset = load_meps_for_location()
+            subset.to_netcdf("subset.nc")
+
+        build_map_overlays(subset, date="2024-05-14", hour="16")
+
+        wind_fig = create_wind_map(subset, altitude_max=3000,x_target=x_target, y_target=y_target)
+        
+
+        # Plot thermal top on a map for a specific time
+        #subset.sel(time=subset.time.min()).thermal_top.plot()
+        sounding_fig = create_sounding(subset, date="2024-05-12", hour=15, x_target=x_target, y_target=y_target)
+
+    
+

requirements.txt

@@ -0,0 +1,19 @@
+plotly>=4.12.0
+requests>=2.24.0
+streamlit>=0.85.1
+pandas>=1.1.3
+geopandas>=0.10.2
+metar>=1.8.0
+python-dateutil>=2.8.1
+#netatmo #>=1.0.7
+numpy
+shapely
+matplotlib
+folium
+streamlit-folium
+windrose
+xarray
+siphon
+netcdf4
+scipy
+bottleneck