From 73bef7fab9f6b374f78845fc87bbaf249298ff6a Mon Sep 17 00:00:00 2001 From: Ben Sherriff Date: Wed, 13 Sep 2023 09:11:54 -0400 Subject: [PATCH] Retrieved metar api logic fixed --- weather-service/Cargo.lock | 10 +-- weather-service/Cargo.toml | 2 +- .../migrations/000001_create_metars/up.sql | 23 +++---- weather-service/src/main.rs | 1 - weather-service/src/metars/model.rs | 69 ++++++++++++++----- 5 files changed, 67 insertions(+), 38 deletions(-) diff --git a/weather-service/Cargo.lock b/weather-service/Cargo.lock index d7f46aa..4032ca2 100644 --- a/weather-service/Cargo.lock +++ b/weather-service/Cargo.lock @@ -119,7 +119,7 @@ dependencies = [ "futures-core", "futures-util", "mio", - "socket2 0.5.3", + "socket2 0.5.4", "tokio", "tracing", ] @@ -180,7 +180,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "smallvec", - "socket2 0.5.3", + "socket2 0.5.4", "time", "url", ] @@ -1521,9 +1521,9 @@ dependencies = [ [[package]] name = "socket2" -version = "0.5.3" +version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e" dependencies = [ "libc", "windows-sys", @@ -1629,7 +1629,7 @@ dependencies = [ "parking_lot", "pin-project-lite", "signal-hook-registry", - "socket2 0.5.3", + "socket2 0.5.4", "tokio-macros", "windows-sys", ] diff --git a/weather-service/Cargo.toml b/weather-service/Cargo.toml index bf254f4..f30d446 100644 --- a/weather-service/Cargo.toml +++ b/weather-service/Cargo.toml @@ -8,7 +8,7 @@ edition = "2021" [dependencies] actix-web = "4.4.0" actix-rt = "2.9.0" -chrono = { version = "0.4.28", features = ["serde"] } +chrono = { version = "0.4.30", features = ["serde"] } dotenv = "0.15.0" diesel = { version = "2.0", features = ["postgres", "r2d2", "uuid", "chrono"] } postgis_diesel = { version = "2.2.1", features = ["serde"] } diff --git a/weather-service/migrations/000001_create_metars/up.sql b/weather-service/migrations/000001_create_metars/up.sql index 724ddfd..6d80b5e 100644 --- a/weather-service/migrations/000001_create_metars/up.sql +++ b/weather-service/migrations/000001_create_metars/up.sql @@ -1,24 +1,23 @@ CREATE TABLE IF NOT EXISTS metars ( - id SERIAL PRIMARY KEY, - icao TEXT NOT NULL, + id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY, raw_text TEXT NOT NULL, station_id TEXT NOT NULL, observation_time TEXT NOT NULL, - latitude INTEGER NOT NULL, - longitude INTEGER NOT NULL, - temp_c DOUBLE PRECISION NOT NULL, - dewpoint_c DOUBLE PRECISION NOT NULL, - wind_dir_degrees INTEGER NOT NULL, - wind_speed_kt INTEGER NOT NULL, - visibility_statute_mi TEXT NOT NULL, - altim_in_hg DOUBLE PRECISION NOT NULL, + latitude DOUBLE PRECISION NOT NULL, + longitude DOUBLE PRECISION NOT NULL, + temp_c DOUBLE PRECISION, + dewpoint_c DOUBLE PRECISION, + wind_dir_degrees TEXT, + wind_speed_kt INTEGER, + visibility_statute_mi TEXT, + altim_in_hg DOUBLE PRECISION, sea_level_pressure_mb DOUBLE PRECISION, wx_string TEXT, flight_category TEXT, three_hr_pressure_tendency_mb DOUBLE PRECISION, - metar_type TEXT, + metar_type TEXT NOT NULL, max_t_c DOUBLE PRECISION, min_t_c DOUBLE PRECISION, precip_in DOUBLE PRECISION, - elevation_m INTEGER + elevation_m INTEGER NOT NULL ); \ No newline at end of file diff --git a/weather-service/src/main.rs b/weather-service/src/main.rs index 9b48c9f..3d1c06e 100644 --- a/weather-service/src/main.rs +++ b/weather-service/src/main.rs @@ -1,4 +1,3 @@ -#[macro_use] extern crate diesel; #[macro_use] extern crate diesel_migrations; diff --git a/weather-service/src/metars/model.rs b/weather-service/src/metars/model.rs index f7fcd72..fa7f577 100644 --- a/weather-service/src/metars/model.rs +++ b/weather-service/src/metars/model.rs @@ -1,7 +1,8 @@ use crate::{error_handler::CustomError, db}; use crate::schema::metars; -use diesel::prelude::*; -use log::{warn, error, debug}; +use diesel::{prelude::*, sql_query}; +use log::{warn, trace}; +use std::collections::HashSet; use std::io::BufRead; use quick_xml::{Reader, events::{Event, BytesStart}, Writer, de::Deserializer}; use serde::{Deserialize, Serialize}; @@ -64,7 +65,7 @@ impl Metar { let mut deserializer = Deserializer::from_str(str); match Self::deserialize(&mut deserializer) { Ok(m) => metars.push(m), - Err(err) => warn!("{}", err) + Err(err) => warn!("Error deserializing; {}", err) }; }, _ => () @@ -106,7 +107,8 @@ impl Metar { } } -#[derive(Serialize, Deserialize, Queryable)] +#[derive(Serialize, Deserialize, Queryable, QueryableByName)] +#[diesel(table_name = metars)] pub struct Metars { pub id: i32, pub raw_text: String, @@ -141,15 +143,44 @@ impl Metars { return Ok(vec![]); } let station_icaos: Vec<&str> = icaos.split(',').collect(); + let station_query: Vec = station_icaos.iter().map(|icao| format!("'{}'", icao.to_string())).collect(); + let mut conn = db::connection()?; - let db_metars: Vec = match metars::table - .filter(metars::station_id.eq_any(station_icaos)) - .order(metars::id.asc()) - .load::(&mut conn) { - Ok(m) => m, - Err(err) => return Err(CustomError { error_status_code: 500, error_message: format!("{}", err) }) - }; - let url = format!("https://beta.aviationweather.gov/cgi-bin/data/metar.php?ids={}&format=xml", icaos); + let mut db_metars: Vec = match sql_query(format!("SELECT DISTINCT ON (station_id) * FROM metars WHERE station_id IN ({}) ORDER BY station_id, observation_time DESC", station_query.join(","))).load(&mut conn) { + Ok(m) => m, + Err(err) => return Err(CustomError { error_status_code: 500, error_message: format!("{}", err) }) + }; + fn get_missing_metar_icaos(db_metars: &Vec, station_icaos: Vec<&str>) -> Vec { + let mut missing_metar_icaos: Vec = vec![]; + let current_time = chrono::Local::now().naive_local().timestamp(); + let db_metars_set: HashSet<&str> = db_metars.iter().map(|icao| icao.station_id.as_str()).collect(); + let station_icaos_set: HashSet<&str> = station_icaos.into_iter().collect(); + for difference in db_metars_set.symmetric_difference(&station_icaos_set) { + missing_metar_icaos.push(difference.to_string()); + } + for metar in db_metars { + match chrono::NaiveDateTime::parse_and_remainder(&metar.observation_time, "%Y-%m-%dT%H:%M:%S") { + Ok((time, _)) => { + if current_time > (time.timestamp() + 3600) { + trace!("{} METAR data is outdated", metar.station_id); + missing_metar_icaos.push(metar.station_id.to_string()); + } + }, + Err(err) => { + warn!("Parsing METAR timestamp failed; {}", err); + missing_metar_icaos.push(metar.station_id.to_string()); + } + }; + } + return missing_metar_icaos; + } + let missing_icaos = get_missing_metar_icaos(&db_metars, station_icaos); + if missing_icaos.is_empty() { + return Ok(db_metars); + } + trace!("Retrieving missing METAR data for {:?}", missing_icaos); + let missing_icaos_string: Vec = missing_icaos.iter().map(|icao| format!("'{}'", icao.to_string())).collect(); + let url = format!("https://beta.aviationweather.gov/cgi-bin/data/metar.php?ids={}&format=xml", missing_icaos_string.join(",")); let metars: Vec = match reqwest::get(url).await { Ok(r) => match r.text().await { Ok(r) => { @@ -171,15 +202,14 @@ impl Metars { vec![] } }; - match diesel::insert_into(metars::table).values(&metars).execute(&mut conn) { - Ok(rows) => debug!("Inserted {} metar rows", rows), - Err(err) => error!("Unable to insert metar data; {}", err) - }; + if metars.len() > 0 { + match diesel::insert_into(metars::table).values(&metars).execute(&mut conn) { + Ok(rows) => trace!("Inserted {} metar rows", rows), + Err(err) => warn!("Unable to insert metar data; {}", err) + }; + } let mut returned_metars: Vec = vec![]; for metar in &metars { - // let _ = diesel::insert_into(metars::table) - // .values(metar) - // .execute(&mut conn); returned_metars.push(Self { id: 0, raw_text: metar.raw_text.to_string(), @@ -213,6 +243,7 @@ impl Metars { elevation_m: metar.elevation_m, }) } + returned_metars.append(&mut db_metars); Ok(returned_metars) } } \ No newline at end of file