Cleaned up metars some, fixed humidity calculation

README.md

@@ -35,7 +35,7 @@ to allow communication within the docker network
 * `ADMIN_EMAIL` - Please change in production environments
 * `ADMIN_PASSWORD` - Please change in production environments
 * `VITE_API_URL` - Change to the FQDN of the URL that is reachable through the internet.
-For example: `https://aviaation.bensherriff.com`
+For example: `https://aviation.bensherriff.com`
 * `__VITE_ADDITIONAL_SERVER_ALLOWED_HOSTS` - Change to the domain of the `VITE_API_URL`.
 For example: `aviation.bensherriff.com`
 
@@ -64,7 +64,10 @@ Metar data is collected from aviationweather.gov.
 The following resources were used to help decode METARS.
 - [Metar Decode Key PDF](https://www.weather.gov/media/wrh/mesowest/metar_decode_key.pdf)
 - [Metar Decode (NPS EDU)](https://met.nps.edu/~bcreasey/mr3222/files/helpful/DecodeMETAR-TAF.html)
-- [Weather Phenomena](http://www.moratech.com/aviation/metar-class/metar-pg9-ww.html)
+- [moratech.com](http://moratech.com/aviation/metar-class/metar.html#INDEX)
 
 ### OpenMapTiles
 [Generate Vector Tiles](https://openmaptiles.org/docs/generate/generate-openmaptiles/)
+
+### Other data
+- https://www.faa.gov/air_traffic/weather/asos

api/migrations/10232024_initial.sql

@@ -52,7 +52,8 @@ CREATE TABLE IF NOT EXISTS metars (
     icao TEXT NOT NULL,
     observation_time TIMESTAMPTZ NOT NULL,
     raw_text TEXT NOT NULL,
-    data JSONB NOT NULL
+    data JSONB NOT NULL,
+    PRIMARY KEY(icao, observation_time)
 );
 
 CREATE INDEX ON metars (observation_time DESC);

api/src/airports/model/airport.rs

@@ -353,7 +353,7 @@ impl Airport {
           Some(
             metars
               .into_iter()
-              .map(|m| (m.station_id.clone(), m))
+              .map(|m| (m.icao.clone(), m))
               .collect::<HashMap<_, _>>(),
           ),
         )

api/src/metars/model.rs

@@ -3,6 +3,8 @@ use crate::{error::ApiResult, db};
 use chrono::{DateTime, Datelike, Utc};
 use std::collections::HashSet;
 use std::env;
+use std::fmt::Display;
+use std::str::FromStr;
 use redis::{AsyncCommands, RedisResult};
 use reqwest::Client;
 use serde::{Deserialize, Serialize};
@@ -12,13 +14,24 @@ const TABLE_NAME: &str = "metars";
 
 #[derive(Serialize, Deserialize, Debug)]
 pub struct Metar {
-  pub station_id: String, // icao
+  pub icao: String,
   pub raw_text: String,
   pub observation_time: DateTime<Utc>,
+  pub flight_category: FlightCategory,
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub report_modifier: Option<ReportModifier>,
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub becoming_change: Option<bool>,
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub no_significant_change: Option<bool>,
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub temporary_change: Option<bool>,
   #[serde(skip_serializing_if = "Option::is_none")]
   pub temp_c: Option<f64>,
   #[serde(skip_serializing_if = "Option::is_none")]
-  pub dewpoint_c: Option<f64>,
+  pub dew_point_c: Option<f64>,
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub estimated_humidity: Option<f64>,
   #[serde(skip_serializing_if = "Option::is_none")]
   pub wind_dir_degrees: Option<String>,
   #[serde(skip_serializing_if = "Option::is_none")]
@@ -31,27 +44,46 @@ pub struct Metar {
   pub visibility_statute_mi: Option<String>,
   pub runway_visual_range: Vec<RunwayVisualRange>,
   #[serde(skip_serializing_if = "Option::is_none")]
-  pub altim_in_hg: Option<f64>,
+  pub altimeter_in_hg: Option<f64>, // inches of mercury units
   #[serde(skip_serializing_if = "Option::is_none")]
   pub sea_level_pressure_mb: Option<f64>,
   pub remarks: Remarks,
   pub weather_phenomena: Vec<String>,
   pub sky_condition: Vec<SkyCondition>,
-  pub flight_category: FlightCategory,
   #[serde(skip_serializing_if = "Option::is_none")]
-  pub three_hr_pressure_tendency_mb: Option<f64>,
+  pub max_temp_c: Option<f64>, // TODO
   #[serde(skip_serializing_if = "Option::is_none")]
-  pub max_t_c: Option<f64>,
+  pub min_temp_c: Option<f64>, // TODO
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub min_t_c: Option<f64>,
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub precip_in: Option<f64>,
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub humidity: Option<f64>,
   #[serde(skip_serializing_if = "Option::is_none")]
   pub density_altitude: Option<f64>,
-  #[serde(skip_serializing_if = "Option::is_none")]
+}
-  pub no_significant_changes: Option<bool>,
+
+#[derive(Serialize, Deserialize, Debug)]
+pub enum ReportModifier {
+  #[serde(rename = "AUTO")]
+  Auto,
+  #[serde(rename = "COR")]
+  Corrected
+}
+
+impl FromStr for ReportModifier {
+  type Err = Error;
+  fn from_str(s: &str) -> Result<Self, Self::Err> {
+    match s {
+      "AUTO" => Ok(ReportModifier::Auto),
+      "COR" => Ok(ReportModifier::Corrected),
+      _ => Err(Error::new(400, format!("Invalid report modifier '{}'", s)))
+    }
+  }
+}
+
+impl Display for ReportModifier {
+  fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+    match self {
+      ReportModifier::Auto => write!(f, "AUTO"),
+      ReportModifier::Corrected => write!(f, "COR"),
+    }
+  }
 }
 
 #[derive(Serialize, Deserialize, Debug)]
@@ -60,9 +92,9 @@ pub struct RunwayVisualRange {
   #[serde(skip_serializing_if = "Option::is_none")]
   pub visibility_ft: Option<String>,
   #[serde(skip_serializing_if = "Option::is_none")]
-  pub variable_visibility_high_ft: Option<String>,
-  #[serde(skip_serializing_if = "Option::is_none")]
   pub variable_visibility_low_ft: Option<String>,
+  #[serde(skip_serializing_if = "Option::is_none")]
+  pub variable_visibility_high_ft: Option<String>,
 }
 
 impl Default for RunwayVisualRange {
@@ -70,8 +102,36 @@ impl Default for RunwayVisualRange {
     RunwayVisualRange {
       runway: "".to_string(),
       visibility_ft: None,
-      variable_visibility_high_ft: None,
       variable_visibility_low_ft: None,
+      variable_visibility_high_ft: None,
+    }
+  }
+}
+
+#[derive(Serialize, Deserialize, Debug)]
+pub enum AutomatedStationType {
+  #[serde(rename = "AO1")]
+  WithoutPrecipitationDiscriminator,
+  #[serde(rename = "AO2")]
+  WithPrecipitationDiscriminator,
+}
+
+impl FromStr for AutomatedStationType {
+  type Err = Error;
+  fn from_str(s: &str) -> Result<Self, Self::Err> {
+    match s {
+      "AO1" => Ok(AutomatedStationType::WithoutPrecipitationDiscriminator),
+      "AO2" => Ok(AutomatedStationType::WithPrecipitationDiscriminator),
+      _ => Err(Error::new(400, format!("Invalid automated station type '{}'", s)))
+    }
+  }
+}
+
+impl Display for AutomatedStationType {
+  fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
+    match self {
+      AutomatedStationType::WithoutPrecipitationDiscriminator => write!(f, "AO1"),
+      AutomatedStationType::WithPrecipitationDiscriminator => write!(f, "AO2"),
     }
   }
 }
@@ -81,25 +141,15 @@ pub struct Remarks {
   #[serde(skip_serializing_if = "Option::is_none")]
   pub peak_wind: Option<PeakWind>,
   #[serde(skip_serializing_if = "Option::is_none")]
-  pub auto: Option<bool>,
+  pub auto_station_type: Option<AutomatedStationType>,
   #[serde(skip_serializing_if = "Option::is_none")]
-  pub auto_station_without_precipication: Option<bool>,
+  pub maintenance_indicator: Option<bool>,
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub auto_station_with_precipication: Option<bool>,
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub maintenance_indicator_on: Option<bool>,
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub corrected: Option<bool>,
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub no_significant_change: Option<bool>,
-  #[serde(skip_serializing_if = "Option::is_none")]
-  pub temporary_change: Option<bool>,
   #[serde(skip_serializing_if = "Option::is_none")]
   pub rvr_missing: Option<bool>,
   #[serde(skip_serializing_if = "Option::is_none")]
-  pub precipication_identifier_information_not_available: Option<bool>,
+  pub precipitation_identifier_information_not_available: Option<bool>,
   #[serde(skip_serializing_if = "Option::is_none")]
-  pub precipication_information_not_available: Option<bool>,
+  pub precipitation_information_not_available: Option<bool>,
   #[serde(skip_serializing_if = "Option::is_none")]
   pub freezing_rain_information_not_available: Option<bool>,
   #[serde(skip_serializing_if = "Option::is_none")]
@@ -122,16 +172,11 @@ impl Default for Remarks {
   fn default() -> Self {
     Remarks {
       peak_wind: None,
-      auto: None,
+      auto_station_type: None,
-      auto_station_without_precipication: None,
+      maintenance_indicator: None,
-      auto_station_with_precipication: None,
-      maintenance_indicator_on: None,
-      corrected: None,
-      no_significant_change: None,
-      temporary_change: None,
       rvr_missing: None,
-      precipication_identifier_information_not_available: None,
+      precipitation_identifier_information_not_available: None,
-      precipication_information_not_available: None,
+      precipitation_information_not_available: None,
       freezing_rain_information_not_available: None,
       thunderstorm_information_not_available: None,
       visibility_at_secondary_location_not_available: None,
@@ -172,31 +217,32 @@ impl Default for Metar {
   fn default() -> Self {
     Self {
       raw_text: "".to_string(),
-      station_id: "".to_string(),
+      icao: "".to_string(),
       observation_time: chrono::DateTime::parse_from_rfc3339("1970-01-01T00:00:00Z")
         .unwrap()
         .with_timezone(&Utc),
+      flight_category: FlightCategory::UNKN,
+      report_modifier: None,
+      no_significant_change: None,
+      temporary_change: None,
+      becoming_change: None,
       temp_c: None,
-      dewpoint_c: None,
+      dew_point_c: None,
       wind_dir_degrees: None,
       wind_speed_kt: None,
       wind_gust_kt: None,
       variable_wind_dir_degrees: None,
       visibility_statute_mi: None,
       runway_visual_range: vec![],
-      altim_in_hg: None,
+      altimeter_in_hg: None,
       sea_level_pressure_mb: None,
       remarks: Remarks::default(),
       weather_phenomena: vec![],
       sky_condition: vec![],
-      flight_category: FlightCategory::UNKN,
+      max_temp_c: None,
-      three_hr_pressure_tendency_mb: None,
+      min_temp_c: None,
-      max_t_c: None,
+      estimated_humidity: None,
-      min_t_c: None,
-      precip_in: None,
-      humidity: None,
       density_altitude: None,
-      no_significant_changes: None,
     }
   }
 }
@@ -221,6 +267,9 @@ impl MetarRow {
         data
       )
       VALUES ($1, $2, $3, $4)
+      ON CONFLICT (icao, observation_time) DO UPDATE SET
+        raw_text = EXCLUDED.raw_text,
+        data = EXCLUDED.data
       "#,
       TABLE_NAME,
     ))
@@ -279,7 +328,7 @@ impl Metar {
     }
 
     // Station Identifier
-    metar.station_id = metar_parts[0].to_string();
+    metar.icao = metar_parts[0].to_string();
     metar_parts.remove(0);
 
     // Date/Time
@@ -338,16 +387,12 @@ impl Metar {
         break;
       }
       // Report Modifiers
-      if !metar_parts.is_empty() && metar_parts[0] == "AUTO" {
+      if !metar_parts.is_empty() && (metar_parts[0] == "AUTO" || metar_parts[0] == "COR") {
-        metar.remarks.auto = Some(true);
+        metar.report_modifier = Some(ReportModifier::from_str(metar_parts[0])?);
-        metar_parts.remove(0);
-      }
-      if !metar_parts.is_empty() && metar_parts[0] == "COR" {
-        metar.remarks.corrected = Some(true);
         metar_parts.remove(0);
       }
       if !metar_parts.is_empty() && metar_parts[0] == "NOSIG" {
-        metar.remarks.no_significant_change = Some(true);
+        metar.no_significant_change = Some(true);
         metar_parts.remove(0);
       }
 
@@ -478,14 +523,13 @@ impl Metar {
             "P{}",
             visibility_whole
               + (visibility_left[1..visibility_left.len()]
-                .parse::<f64>()
+                .parse::<f64>()?
-                .unwrap()
                 / visibility_right)
           )
         } else {
           format!(
             "{}",
-            visibility_whole + (visibility_left.parse::<f64>().unwrap() / visibility_right)
+            visibility_whole + (visibility_left.parse::<f64>()? / visibility_right)
           )
         };
         metar.visibility_statute_mi = Some(visibility);
@@ -496,7 +540,7 @@ impl Metar {
         if &visibility[0..4] == "9999" {
           metar.visibility_statute_mi = Some("P10".to_string());
         } else {
-          let visibility = visibility[0..4].parse::<f64>().unwrap() * 0.000621371;
+          let visibility = visibility[0..4].parse::<f64>()? * 0.000621371;
           metar.visibility_statute_mi = Some(format!("{:.2}", visibility));
         }
       }
@@ -614,7 +658,7 @@ impl Metar {
           dewpoint_c = temp_parts[1];
         }
         if temp_c.starts_with("M") {
-          metar.temp_c = Some(temp_c[1..temp_c.len()].parse::<f64>().unwrap() * -1.0);
+          metar.temp_c = Some(temp_c[1..temp_c.len()].parse::<f64>()? * -1.0);
         } else if !temp_c.is_empty() {
           metar.temp_c = match temp_c.parse::<f64>() {
             Ok(t) => Some(t),
@@ -625,9 +669,9 @@ impl Metar {
           };
         }
         if dewpoint_c.starts_with("M") {
-          metar.dewpoint_c = Some(dewpoint_c[1..dewpoint_c.len()].parse::<f64>().unwrap() * -1.0);
+          metar.dew_point_c = Some(dewpoint_c[1..dewpoint_c.len()].parse::<f64>()? * -1.0);
         } else if !dewpoint_c.is_empty() {
-          metar.dewpoint_c = match dewpoint_c.parse::<f64>() {
+          metar.dew_point_c = match dewpoint_c.parse::<f64>() {
             Ok(d) => Some(d),
             Err(err) => {
               log::warn!("Unable to parse dewpoint in {}: {}", dewpoint_c, err);
@@ -642,7 +686,7 @@ impl Metar {
       if !metar_parts.is_empty() && altim_re.is_match(metar_parts[0]) {
         let altim = metar_parts[0];
         metar_parts.remove(0);
-        metar.altim_in_hg = Some(altim[1..altim.len()].parse::<f64>().unwrap() / 100.0);
+        metar.altimeter_in_hg = Some(altim[1..altim.len()].parse::<f64>()? / 100.0);
       }
 
       // Pressure
@@ -650,18 +694,24 @@ impl Metar {
       if !metar_parts.is_empty() && pressure_re.is_match(metar_parts[0]) {
         let pressure = metar_parts[0];
         metar_parts.remove(0);
-        metar.sea_level_pressure_mb = Some(pressure[1..pressure.len()].parse::<f64>().unwrap());
+        metar.sea_level_pressure_mb = Some(pressure[1..pressure.len()].parse::<f64>()?);
       }
 
-      // Temporary Change
+      // Trend forecast - becoming change
-      if !metar_parts.is_empty() && metar_parts[0] == "TEMPO" {
+      if !metar_parts.is_empty() && metar_parts[0] == "BECMG" {
-        metar.remarks.temporary_change = Some(true);
+        metar.becoming_change = Some(true);
         metar_parts.remove(0);
       }
 
-      // No significant changes
+      // Trend forecast - temporary change
+      if !metar_parts.is_empty() && metar_parts[0] == "TEMPO" {
+        metar.temporary_change = Some(true);
+        metar_parts.remove(0);
+      }
+
+      // Trend forecast - No significant change
       if !metar_parts.is_empty() && metar_parts[0] == "NOSIG" {
-        metar.no_significant_changes = Some(true);
+        metar.no_significant_change = Some(true);
         metar_parts.remove(0);
       }
 
@@ -676,12 +726,10 @@ impl Metar {
           let hourly_temp_re = regex::Regex::new(r"^T[01][0-9]{3}[01][0-9]{3}$").unwrap();
           let remark = metar_parts[0];
           metar_parts.remove(0);
-          if remark == "AO1" {
+          if remark == "AO1" || remark == "AO2" {
-            metar.remarks.auto_station_without_precipication = Some(true);
+            metar.remarks.auto_station_type = Some(AutomatedStationType::from_str(remark)?);
-          } else if remark == "AO2" {
-            metar.remarks.auto_station_with_precipication = Some(true);
           } else if remark == "$" {
-            metar.remarks.maintenance_indicator_on = Some(true);
+            metar.remarks.maintenance_indicator = Some(true);
           } else if remark == "PK" && metar_parts.len() >= 2 && metar_parts[0] == "WND" {
             metar_parts.remove(0);
             let string = metar_parts[0];
@@ -715,13 +763,13 @@ impl Metar {
               ));
             }
           } else if remark == "PNO" {
-            metar.remarks.precipication_information_not_available = Some(true);
+            metar.remarks.precipitation_information_not_available = Some(true);
           } else if remark == "RVRNO" {
             metar.remarks.rvr_missing = Some(true);
           } else if remark == "PWINO" {
             metar
               .remarks
-              .precipication_identifier_information_not_available = Some(true);
+              .precipitation_identifier_information_not_available = Some(true);
           } else if remark == "FZRANO" {
             metar.remarks.freezing_rain_information_not_available = Some(true);
           } else if remark == "TSNO" {
@@ -739,7 +787,7 @@ impl Metar {
               .sky_condition_at_secondary_location_not_available = Some(location.to_string());
           } else if slp_re.is_match(remark) {
             let slp = slp_re.captures(remark).unwrap();
-            let sea_level_pressure = slp[1].parse::<f64>().unwrap();
+            let sea_level_pressure = slp[1].parse::<f64>()?;
             if sea_level_pressure > 500.0 {
               metar.sea_level_pressure_mb = Some((sea_level_pressure / 10.0) + 900.0);
             } else {
@@ -759,9 +807,9 @@ impl Metar {
             let dewpoint = &remark[6..9];
             if let Ok(d) = dewpoint.parse::<f64>() {
               if dewpoint_negation == "0" {
-                metar.dewpoint_c = Some(d / 10.0);
+                metar.dew_point_c = Some(d / 10.0);
               } else {
-                metar.dewpoint_c = Some(d / 10.0 * -1.0);
+                metar.dew_point_c = Some(d / 10.0 * -1.0);
               }
             }
           }
@@ -786,9 +834,9 @@ impl Metar {
       let visibility = match &metar.visibility_statute_mi {
         Some(v) => {
           if v.starts_with("M") || v.starts_with("P") {
-            v[1..v.len()].parse::<f64>().unwrap()
+            v[1..v.len()].parse::<f64>()?
           } else {
-            v.parse::<f64>().unwrap()
+            v.parse::<f64>()?
           }
         }
         None => 5.0, // Assume VFR if no visibility is present
@@ -820,10 +868,18 @@ impl Metar {
       }
     }
 
-    // Calculate estimated humidity
+    // Calculate estimated humidity using the magnus formula
-    if metar.temp_c.is_some() && metar.dewpoint_c.is_some() {
+    if metar.temp_c.is_some() && metar.dew_point_c.is_some() {
-      let estimated_humidity = 100.0 - ((metar.temp_c.unwrap() - metar.dewpoint_c.unwrap()) * 5.0);
+      let temp = metar.temp_c.unwrap();
-      metar.humidity = Some(estimated_humidity);
+      let dew_point = metar.dew_point_c.unwrap();
+      let a: f64 = 17.625;
+      let b: f64 = 243.04;
+      let exponent_temp = a * temp / (b + temp);
+      let exponent_dew = a * dew_point / (b + dew_point);
+      let mut estimated_humidity = 100.0 * (exponent_dew.exp() / exponent_temp.exp());
+      // Round to 3 decimal places
+      estimated_humidity = (estimated_humidity * 1000.0).round() / 1000.0;
+      metar.estimated_humidity = Some(estimated_humidity);
     }
 
     // Calculate estimated density
@@ -841,7 +897,7 @@ impl Metar {
     let current_time = chrono::Local::now().naive_local().and_utc().timestamp();
     let db_metars_set: HashSet<&str> = db_metars
       .iter()
-      .map(|icao| icao.station_id.as_str())
+      .map(|icao| icao.icao.as_str())
       .collect();
     let station_icaos_set: HashSet<&str> = station_icaos.iter().map(|s| s.as_str()).collect();
     for difference in db_metars_set.symmetric_difference(&station_icaos_set) {
@@ -853,8 +909,8 @@ impl Metar {
       .unwrap_or(3000);
     for metar in db_metars {
       if current_time > (metar.observation_time.timestamp() + time_offset) {
-        log::trace!("{} METAR data is outdated", metar.station_id);
+        log::trace!("{} METAR data is outdated", metar.icao);
-        missing_metar_icaos.push(metar.station_id.to_string());
+        missing_metar_icaos.push(metar.icao.to_string());
       }
     }
     missing_metar_icaos
@@ -909,7 +965,7 @@ impl Metar {
   fn to_db(&self) -> ApiResult<MetarRow> {
     let data = serde_json::to_value(self)?;
     Ok(MetarRow {
-      icao: self.station_id.clone(),
+      icao: self.icao.clone(),
       observation_time: self.observation_time,
       raw_text: self.raw_text.clone(),
       data,
@@ -981,7 +1037,7 @@ impl Metar {
         if missing_icao_list.len() > 0 {
           // Insert missing METARs
           for missing_metar in &missing_icao_list {
-            let _: RedisResult<()> = conn.set(&missing_metar.station_id, true).await;
+            let _: RedisResult<()> = conn.set(&missing_metar.icao, true).await;
             missing_metar.insert().await?;
           }
           metars.append(&mut missing_icao_list)