Working on fixing metars, airport layout, etc

This commit is contained in:
2025-06-02 16:54:53 -04:00
parent 7dedc7a8dc
commit 263c33fd5a
24 changed files with 691 additions and 510 deletions

View File

@@ -2,8 +2,9 @@ use crate::airports::{Airport, UpdateAirport};
use crate::error::Error;
use crate::http_client::HttpClient;
use crate::metars::MetarCheck;
use crate::metars::utils::parse_metar_time;
use crate::{db, error::ApiResult};
use chrono::{DateTime, Datelike, NaiveDate, Utc};
use chrono::{DateTime, Utc};
use flate2::read::GzDecoder;
use reqwest::header::ETAG;
use serde::{Deserialize, Serialize};
@@ -13,6 +14,7 @@ use std::fmt::Display;
use std::io::{Cursor, Read};
use std::str::FromStr;
use std::sync::OnceLock;
use sqlx::{Postgres, QueryBuilder};
use utoipa::ToSchema;
static TIME_OFFSET: OnceLock<i64> = OnceLock::new();
@@ -302,6 +304,39 @@ impl MetarRow {
Ok(())
}
async fn insert_all(metars: Vec<Metar>) -> ApiResult<()> {
let pool = db::pool();
let chunk_size = 1000;
for chunk in metars.chunks(chunk_size) {
let mut query_builder: QueryBuilder<Postgres> = QueryBuilder::new(
format!("INSERT INTO {} (icao, observation_time, raw_text, data) ", TABLE_NAME));
query_builder.push_values(chunk, |mut b, metar | {
let row: Self = match metar.to_row() {
Ok(row) => row,
Err(e) => {
log::warn!("Failed to serialize METAR data: {}", e);
return;
}
};
b.push_bind(row.icao)
.push_bind(row.observation_time)
.push_bind(row.raw_text)
.push_bind(row.data);
});
query_builder.push(
" ON CONFLICT (icao, observation_time) DO UPDATE SET \
raw_text = EXCLUDED.raw_text, \
data = EXCLUDED.data",
);
let query = query_builder.build();
query.execute(pool).await?;
}
Ok(())
}
}
impl Metar {
@@ -342,7 +377,7 @@ impl Metar {
));
}
// Remove METAR at start of text
// Remove METAR at the start of the text
if metar_parts[0].to_string() == "METAR".to_string() {
metar_parts.remove(0);
}
@@ -354,10 +389,18 @@ impl Metar {
// Date/Time
let observation_time = metar_parts[0];
metar_parts.remove(0);
let observation_time = Self::parse_time(observation_time)?;
metar.observation_time = match chrono::DateTime::parse_from_rfc3339(&observation_time) {
Ok(datetime) => datetime.with_timezone(&Utc),
Err(err) => return Err(err.into()),
match parse_metar_time(observation_time) {
Ok(observation_time) => {
metar.observation_time = match chrono::DateTime::parse_from_rfc3339(&observation_time) {
Ok(datetime) => datetime.with_timezone(&Utc),
Err(err) => return Err(err.into()),
};
},
Err(err) => {
return Err(Error::new(
err.status,
format!("Unexpected observation time field '{}': {}; {}", observation_time, metar_string, err)));
}
};
loop {
@@ -375,9 +418,8 @@ impl Metar {
}
// Wind Direction and Speed
let wind_re = regex::Regex::new(r"^(?:[0-9]{3}|VRB)[0-9]{2}(?:KT|MPS)$").unwrap();
let wind_gust_re =
regex::Regex::new(r"^(?:[0-9]{3}|VRB)[0-9]{2}G[0-9]{2}(?:KT|MPS)$").unwrap();
let wind_re = regex::Regex::new(r"^(?:[0-9]{3}|VRB)[0-9]{2}(?:KT|MPS)$")?;
let wind_gust_re = regex::Regex::new(r"^(?:[0-9]{3}|VRB)[0-9]{2}G[0-9]{2}(?:KT|MPS)$")?;
// Handle input error where there is a space between the numbers and units
let mut value: Option<String> = None;
if metar_parts.len() >= 2
@@ -411,9 +453,9 @@ impl Metar {
let mut wind_speed_kt = wind[3..5].to_string();
// Convert m/s to kt
if wind.len() == 8 {
wind_speed_kt = (wind_speed_kt.parse::<f64>().unwrap() * 1.94384).to_string();
wind_speed_kt = (wind_speed_kt.parse::<f64>()? * 1.94384).to_string();
}
metar.wind_speed_kt = Some(wind_speed_kt.parse::<f64>().unwrap());
metar.wind_speed_kt = Some(wind_speed_kt.parse::<f64>()?);
} else if wind_gust_re.is_match(&wind) {
let wind_dir_degrees = &wind[0..3];
metar.wind_dir_degrees = Some(wind_dir_degrees.to_string());
@@ -421,26 +463,26 @@ impl Metar {
let mut wind_gust_kt = wind[6..8].to_string();
// Convert m/s to kt
if wind.len() == 9 {
wind_speed_kt = (wind_speed_kt.parse::<f64>().unwrap() * 1.94384).to_string();
wind_gust_kt = (wind_gust_kt.parse::<f64>().unwrap() * 1.94384).to_string();
wind_speed_kt = (wind_speed_kt.parse::<f64>()? * 1.94384).to_string();
wind_gust_kt = (wind_gust_kt.parse::<f64>()? * 1.94384).to_string();
}
metar.wind_speed_kt = Some(wind_speed_kt.parse::<f64>().unwrap());
metar.wind_gust_kt = Some(wind_gust_kt.parse::<f64>().unwrap());
metar.wind_speed_kt = Some(wind_speed_kt.parse::<f64>()?);
metar.wind_gust_kt = Some(wind_gust_kt.parse::<f64>()?);
}
}
None => {}
}
// Variable Wind Direction
let variable_wind_re = regex::Regex::new(r"^[0-9]{3}V[0-9]{3}$").unwrap();
let variable_wind_re = regex::Regex::new(r"^[0-9]{3}V[0-9]{3}$")?;
if !metar_parts.is_empty() && variable_wind_re.is_match(metar_parts[0]) {
metar.variable_wind_dir_degrees = Some(metar_parts[0].to_string());
metar_parts.remove(0);
}
// Visibility
let visibility_re = regex::Regex::new(r"^M?(?:[0-9]+|[0-9]+/[0-9]+)SM$").unwrap();
let visibility_re_m = regex::Regex::new(r"^[0-9]{4}(:?N|NE|NW|S|SE|SW)?$").unwrap();
let visibility_re = regex::Regex::new(r"^M?(?:[0-9]+|[0-9]+/[0-9]+)SM$")?;
let visibility_re_m = regex::Regex::new(r"^[0-9]{4}(:?N|NE|NW|S|SE|SW)?$")?;
if !metar_parts.is_empty() && visibility_re.is_match(metar_parts[0]) {
let visibility_str = &metar_parts[0][0..metar_parts[0].len() - 2];
metar_parts.remove(0);
@@ -474,59 +516,68 @@ impl Metar {
metar_parts.remove(0);
let visibility_parts: Vec<&str> = metar_parts[0].split("/").collect();
metar_parts.remove(0);
let visibility_left = visibility_parts[0];
// Parse the right-hand of visibility, with or without an SM suffix
let visibility_right_string = match visibility_parts[1].strip_suffix("SM") {
Some(s) => s,
None => {
if visibility_parts[1].chars().all(|c| c.is_numeric() || c == '.') {
visibility_parts[1]
} else {
log::warn!(
"Skipping invalid visibility field '{}' ({})",
metar_parts[0],
metar_string
);
continue;
if visibility_parts.len() == 1 {
metar.visibility_statute_mi = Some(visibility_parts[0].to_string());
} else if visibility_parts.len() == 2 {
let visibility_left = visibility_parts[0];
// Parse the right-hand of visibility, with or without an SM suffix
let visibility_right_string = match visibility_parts[1].strip_suffix("SM") {
Some(s) => s,
None => {
if visibility_parts[1]
.chars()
.all(|c| c.is_numeric() || c == '.')
{
visibility_parts[1]
} else {
log::warn!(
"Skipping unexpected visibility field '{:?}' ({})",
visibility_parts,
metar_string
);
continue;
}
}
};
let visibility_right = visibility_right_string.parse::<f64>()?;
let visibility = if visibility_left.starts_with("M") {
format!(
"M{}",
visibility_whole
+ (visibility_left[1..visibility_left.len()].parse::<f64>()? / visibility_right)
)
} else if visibility_left.starts_with("P") {
format!(
"P{}",
visibility_whole
+ (visibility_left[1..visibility_left.len()].parse::<f64>()? / visibility_right)
)
} else {
format!(
"{}",
visibility_whole + (visibility_left.parse::<f64>()? / visibility_right)
)
};
metar.visibility_statute_mi = Some(visibility);
} else if !metar_parts.is_empty() && visibility_re_m.is_match(metar_parts[0]) {
// Convert meters to statute miles
let visibility = metar_parts[0];
metar_parts.remove(0);
if &visibility[0..4] == "9999" {
metar.visibility_statute_mi = Some("P10".to_string());
} else {
let visibility = visibility[0..4].parse::<f64>()? * 0.000621371;
metar.visibility_statute_mi = Some(format!("{:.2}", visibility));
}
};
let visibility_right = visibility_right_string.parse::<f64>()?;
let visibility = if visibility_left.starts_with("M") {
format!(
"M{}",
visibility_whole
+ (visibility_left[1..visibility_left.len()].parse::<f64>()? / visibility_right)
)
} else if visibility_left.starts_with("P") {
format!(
"P{}",
visibility_whole
+ (visibility_left[1..visibility_left.len()].parse::<f64>()? / visibility_right)
)
} else {
format!(
"{}",
visibility_whole + (visibility_left.parse::<f64>()? / visibility_right)
)
};
metar.visibility_statute_mi = Some(visibility);
} else if !metar_parts.is_empty() && visibility_re_m.is_match(metar_parts[0]) {
// Convert meters to statute miles
let visibility = metar_parts[0];
metar_parts.remove(0);
if &visibility[0..4] == "9999" {
metar.visibility_statute_mi = Some("P10".to_string());
} else {
let visibility = visibility[0..4].parse::<f64>()? * 0.000621371;
metar.visibility_statute_mi = Some(format!("{:.2}", visibility));
log::warn!("Skipping unexpected visibility field '{}' ({})", metar_parts[0], metar_string);
}
}
// Runway Visual Range
let rvr_re = regex::Regex::new(r"^R[0-9]{1,3}(?:L|R|C)?/[PM]?[0-9]{4}FT$").unwrap();
let rvr_re = regex::Regex::new(r"^R[0-9]{1,3}(?:L|R|C)?/[PM]?[0-9]{4}FT$")?;
let variable_rvr_re =
regex::Regex::new(r"^R[0-9]{1,3}(?:L|R|C)?/[PM]?[0-9]{4}V[PM]?[0-9]{4}FT$").unwrap();
regex::Regex::new(r"^R[0-9]{1,3}(?:L|R|C)?/[PM]?[0-9]{4}V[PM]?[0-9]{4}FT$")?;
while !metar_parts.is_empty()
&& (rvr_re.is_match(metar_parts[0]) || variable_rvr_re.is_match(metar_parts[0]))
{
@@ -567,63 +618,10 @@ impl Metar {
metar_parts.remove(0);
}
// Sky Condition
if !metar_parts.is_empty() && metar_parts[0] == "CAVOK" {
metar.sky_condition.push(SkyCondition {
sky_cover: "CLR".to_string(),
cloud_base_ft_agl: None,
significant_convective_clouds: None,
});
metar_parts.remove(0);
}
let sky_condition_re =
regex::Regex::new(r"^(?:CLR|SKC|NSC|NCD|(?:FEW|SCT|BKN|OVC|VV)([0-9/]{3})?(?:CB|TCU)?)(?:///)?$")
.unwrap();
while !metar_parts.is_empty() && sky_condition_re.is_match(metar_parts[0]) {
let mut sky_condition_string = metar_parts[0];
metar_parts.remove(0);
if sky_condition_string.ends_with("///") {
sky_condition_string = &sky_condition_string[..sky_condition_string.len() - 3];
}
let mut sky_condition = SkyCondition::default();
let mut vv_offset = 0;
if &sky_condition_string[0..2] == "VV" {
sky_condition.sky_cover = "VV".to_string();
vv_offset = 1;
} else {
sky_condition.sky_cover = sky_condition_string[0..3].to_string();
}
if sky_condition_string.len() > 3 - vv_offset {
// Parse out the next three digits
let cloud_base_ft_agl = &sky_condition_string[3 - vv_offset..6 - vv_offset];
if cloud_base_ft_agl == "///" {
sky_condition.cloud_base_ft_agl = None;
} else {
sky_condition.cloud_base_ft_agl = match cloud_base_ft_agl.parse::<i32>() {
Ok(c) => Some(c * 100),
Err(err) => {
log::warn!(
"Unable to parse cloud base in {}: {}",
sky_condition_string,
err
);
None
}
};
}
if sky_condition_string.len() > 6 - vv_offset {
// Parse out the next two digits
let scc = &sky_condition_string[6 - vv_offset..8 - vv_offset];
sky_condition.significant_convective_clouds = Some(scc.to_string());
}
}
metar.sky_condition.push(sky_condition);
}
metar.parse_sky_condition(&mut metar_parts);
// Temperature and Dewpoint
let temp_re = regex::Regex::new(r"^(?:M?[0-9]{2})?/(?:M?[0-9]{2})?$").unwrap();
let temp_re = regex::Regex::new(r"^(?:M?[0-9]{2})?/(?:M?[0-9]{2})?$")?;
if !metar_parts.is_empty() && temp_re.is_match(metar_parts[0]) {
let temp_string = metar_parts[0];
metar_parts.remove(0);
@@ -665,7 +663,7 @@ impl Metar {
}
// Altimeter
let altim_re = regex::Regex::new(r"^A[0-9]{4}$").unwrap();
let altim_re = regex::Regex::new(r"^A[0-9]{4}$")?;
if !metar_parts.is_empty() && altim_re.is_match(metar_parts[0]) {
let altim = metar_parts[0];
metar_parts.remove(0);
@@ -673,7 +671,7 @@ impl Metar {
}
// Pressure
let pressure_re = regex::Regex::new(r"^Q[0-9]{4}$").unwrap();
let pressure_re = regex::Regex::new(r"^Q[0-9]{4}$")?;
if !metar_parts.is_empty() && pressure_re.is_match(metar_parts[0]) {
let pressure = metar_parts[0];
metar_parts.remove(0);
@@ -705,8 +703,8 @@ impl Metar {
if metar_parts.is_empty() {
break;
}
let slp_re = regex::Regex::new(r"^SLP([0-9]{3})$").unwrap();
let hourly_temp_re = regex::Regex::new(r"^T[01][0-9]{3}[01][0-9]{3}$").unwrap();
let slp_re = regex::Regex::new(r"^SLP([0-9]{3})$")?;
let hourly_temp_re = regex::Regex::new(r"^T[01][0-9]{3}[01][0-9]{3}$")?;
let remark = metar_parts[0];
metar_parts.remove(0);
if remark == "AO1" || remark == "AO2" {
@@ -801,7 +799,7 @@ impl Metar {
// Skip unexpected fields
if !metar_parts.is_empty() {
log::warn!(
log::trace!(
"Skipping unexpected field: '{}' ({})",
metar_parts[0],
metar_string
@@ -909,76 +907,68 @@ impl Metar {
Ok(metar)
}
fn parse_time(observation_time: &str) -> ApiResult<String> {
if observation_time.len() != 7 {
return Err(Error::new(
500,
format!("Unable to parse observation time in {}", observation_time),
));
fn parse_sky_condition(&mut self, metar_parts: &mut Vec<&str>) {
// Check if sky condition is CAVOK
if !metar_parts.is_empty() && metar_parts[0] == "CAVOK" {
self.sky_condition.push(SkyCondition {
sky_cover: "CLR".to_string(),
cloud_base_ft_agl: None,
significant_convective_clouds: None,
});
metar_parts.remove(0);
}
let observation_day = match observation_time[0..2].parse::<u32>() {
Ok(day) => day,
Err(err) => return Err(err.into()),
};
let observation_hour = match observation_time[2..4].parse::<u32>() {
Ok(hour) => hour,
Err(err) => return Err(err.into()),
};
let observation_minute = match observation_time[4..6].parse::<u32>() {
Ok(minute) => minute,
Err(err) => return Err(err.into()),
};
let current_time = Utc::now().naive_utc();
let current_year = current_time.year();
let current_month = current_time.month();
let candidate_date = NaiveDate::from_ymd_opt(current_year, current_month, observation_day)
.ok_or_else(|| {
Error::new(
500,
format!(
"Invalid date with day {} for current month",
observation_day
),
)
})?;
let candidate_date = match candidate_date.and_hms_opt(observation_hour, observation_minute, 0) {
Some(date) => date,
None => {
return Err(Error::new(
500,
format!(
"Invalid time for time '{}': hour {}, minute {}",
observation_time, observation_hour, observation_minute
),
));
let sky_condition_re = regex::Regex::new(
r"^(?:CLR|SKC|NSC|NCD|(?:FEW|SCT|BKN|OVC|VV)([0-9/]{3})?(?:CB|TCU)?)(?:///)?$",
)
.unwrap();
while !metar_parts.is_empty() && sky_condition_re.is_match(metar_parts[0]) {
// Get the next METAR part
let mut sky_condition_string = metar_parts[0];
metar_parts.remove(0);
// Remove trailing slashes
if sky_condition_string.ends_with("///") {
sky_condition_string = &sky_condition_string[..sky_condition_string.len() - 3];
}
};
let obs_datetime = if candidate_date > current_time {
// Subtract one month. (Handle year rollover carefully.)
let (month, year) = if current_month == 1 {
(12, current_year - 1)
let mut sky_condition = SkyCondition::default();
// Handle sky cover and optionally vertical visibility
let mut vv_offset = 0;
if &sky_condition_string[0..2] == "VV" {
sky_condition.sky_cover = "VV".to_string();
vv_offset = 1;
} else {
(current_month - 1, current_year)
};
sky_condition.sky_cover = sky_condition_string[0..3].to_string();
}
if sky_condition_string.len() > 3 - vv_offset {
if sky_condition_string.len() < 6 - vv_offset {
// Parse out the significant convective clouds
let scc = &sky_condition_string[3 - vv_offset..];
sky_condition.significant_convective_clouds = Some(scc.to_string());
} else {
// Parse out the next three digits
let cloud_base_ft_agl = &sky_condition_string[3 - vv_offset..6 - vv_offset];
sky_condition.cloud_base_ft_agl = match cloud_base_ft_agl.parse::<i32>() {
Ok(c) => Some(c * 100),
Err(err) => {
log::warn!(
"Unable to parse cloud base in {}: {}",
sky_condition_string,
err
);
None
}
};
let adjusted_date =
NaiveDate::from_ymd_opt(year, month, observation_day).ok_or_else(|| {
Error::new(
500,
format!(
"Invalid date with day {} for month {}",
observation_day, month
),
)
})?;
adjusted_date
.and_hms_opt(observation_hour, observation_minute, 0)
.unwrap()
} else {
candidate_date
};
Ok(obs_datetime.format("%Y-%m-%dT%H:%M:00Z").to_string())
// Parse out the significant convective clouds
let scc = &sky_condition_string[6 - vv_offset..];
sky_condition.significant_convective_clouds = Some(scc.to_string());
}
}
self.sky_condition.push(sky_condition);
}
}
pub async fn get_cached_remote_metars(
@@ -1004,7 +994,7 @@ impl Metar {
let mut output: Vec<Metar> = Vec::new();
for line in text.lines() {
// Split off first column
// Split off the first column
let raw_text = line.splitn(2, ',').next().unwrap();
match Metar::parse(raw_text) {
Ok(m) => output.push(m),
@@ -1017,7 +1007,7 @@ impl Metar {
match new_etag {
Some(etag) => Ok((output, etag)),
None => match etag {
Some(etag) => Ok((output, etag)),
Some(etag) => Ok((output, etag.to_string())),
None => Ok((output, String::new())),
},
}
@@ -1212,9 +1202,7 @@ impl Metar {
log::warn!("Unable to get cached remote METAR data; {}", err);
(vec![], String::new())
});
for remote_metar in remote_metars.clone() {
remote_metar.insert().await?;
}
MetarRow::insert_all(remote_metars).await?;
Ok(etag)
}
@@ -1234,51 +1222,20 @@ impl Metar {
#[cfg(test)]
mod tests {
use super::*;
use chrono::NaiveDateTime;
#[test]
fn test_parse_time() {
for day in 1..=31 {
for hour in 0..24 {
for minute in 0..60 {
// METAR form "DDHHMMZ"
let obs_time = format!("{:02}{:02}{:02}Z", day, hour, minute);
let result = Metar::parse_time(&obs_time);
match result {
Ok(datetime_str) => {
// "YYYY-MM-DDTHH:MM:00Z"
assert_eq!(
datetime_str.len(),
20,
"Unexpected length for input {} yielded {}",
obs_time,
datetime_str
);
// Remove the trailing 'Z' and parse
let trimmed = &datetime_str[..19];
NaiveDateTime::parse_from_str(trimmed, "%Y-%m-%dT%H:%M:%S").unwrap_or_else(|e| {
panic!(
"Parsing '{}' from input {} failed: {}",
trimmed, obs_time, e
)
});
}
Err(_err) => {}
}
}
}
}
}
#[tokio::test]
async fn test_metar() {
let mut metar_string = "METAR KABC 121755Z AUTO 21016G24KT 180V240 1SM R11/P6000FT -RA BR BKN015 OVC025 06/04 A2990
RMK AO2 PK WND 20032/25 WSHFT 1715 VIS 3/4V1 1/2 VIS 3/4 RWY11 RAB07 CIG 013V017 CIG 017 RWY11 PRESFR
SLP125 P0003 60009 T00640036 10066 21012 58033 TSNO $".to_string();
async fn test_metar_parse() {
let mut metar_string = "METAR KABC 121755Z AUTO 21016G24KT 180V240 1SM R11/P6000FT \
-RA BR BKN015 OVC025 06/04 A2990 RMK AO2 PK WND 20032/25 WSHFT 1715 VIS 3/4V1 1/2 VIS 3/4 \
RWY11 RAB07 CIG 013V017 CIG 017 RWY11 PRESFR SLP125 P0003 60009 T00640036 10066 21012 58033 \
TSNO $"
.to_string();
let metar = Metar::parse(&metar_string).unwrap();
dbg!(&metar.observation_time);
metar_string = "KMIA 090053Z 33004KT 10SM FEW015 FEW024 SCT075 SCT250 25/22 A2990 RMK AO2 SLP126 T02500217 $".to_string();
metar_string = "KMIA 090053Z 33004KT 10SM FEW015 FEW024 SCT075 SCT250 25/22 A2990 RMK AO2 \
SLP126 T02500217 $"
.to_string();
let metar = Metar::parse(&metar_string).unwrap();
dbg!(&metar.observation_time);
@@ -1288,12 +1245,24 @@ SLP125 P0003 60009 T00640036 10066 21012 58033 TSNO $".to_string();
let metar = Metar::parse(&metar_string).unwrap();
dbg!(&metar.observation_time);
metar_string = "KHEF 092356Z 13009KT 10SM CLR 08/M03 A3022 RMK AO2 SLP239 6//// T00831033 10133 20078 53002 PNO $".to_string();
metar_string = "KHEF 092356Z 13009KT 10SM CLR 08/M03 A3022 RMK AO2 SLP239 6//// T00831033 \
10133 20078 53002 PNO $"
.to_string();
let metar = Metar::parse(&metar_string).unwrap();
dbg!(&metar.observation_time);
metar_string = "KSLK 162351Z AUTO VRB03KT 1SM -SN BR FEW007 OVC014 00/M02 A2974 RMK AO2 SLP090 P0001 60004 T00001017 10000 21011 53026".to_string();
metar_string = "KSLK 162351Z AUTO VRB03KT 1SM -SN BR FEW007 OVC014 00/M02 A2974 RMK AO2 \
SLP090 P0001 60004 T00001017 10000 21011 53026"
.to_string();
let metar = Metar::parse(&metar_string).unwrap();
dbg!(&metar.observation_time);
metar_string = "KABC 121755Z AUTO 21016G24KT 180V240 1SM R11/P6000FT -RA BR BKN015 OVC025 \
SCTCB FEW123TCU 06/04 A2990 RMK AO2 PK WND 20032/25 WSHFT 1715 VIS 3/4V1 1/2 VIS 3/4 RWY11 \
RAB07 CIG 013V017 CIG 017 RWY11 PRESFR SLP125 P0003 60009 T00640036 10066 21012 58033 TSNO $"
.to_string();
let metar = Metar::parse(&metar_string).unwrap();
dbg!(&metar.observation_time);
dbg!(&metar.sky_condition);
}
}