bsherriff/aviation
1
1
Fork 0
Code Issues 10 Pull Requests Actions Packages Projects 1 Releases Wiki Activity

Retrieved metar api logic fixed

Browse Source
This commit is contained in:
Ben Sherriff
2023-09-13 09:11:54 -04:00
parent e8462665b0
commit 73bef7fab9
5 changed files with 67 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
weather-service/Cargo.lock generated
View File

@@ -119,7 +119,7 @@ dependencies = [
"futures-core",
"futures-util",
"mio",
"socket2 0.5.3",
"socket2 0.5.4",
"tokio",
"tracing",
]
@@ -180,7 +180,7 @@ dependencies = [
"serde_json",
"serde_urlencoded",
"smallvec",
"socket2 0.5.3",
"socket2 0.5.4",
"time",
"url",
]
@@ -1521,9 +1521,9 @@ dependencies = [
[[package]]
name = "socket2"
version = "0.5.3"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877"
checksum = "4031e820eb552adee9295814c0ced9e5cf38ddf1e8b7d566d6de8e2538ea989e"
dependencies = [
"libc",
"windows-sys",
@@ -1629,7 +1629,7 @@ dependencies = [
"parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2 0.5.3",
"socket2 0.5.4",
"tokio-macros",
"windows-sys",
]

2
weather-service/Cargo.toml
View File

@@ -8,7 +8,7 @@ edition = "2021"
[dependencies]
actix-web = "4.4.0"
actix-rt = "2.9.0"
chrono = { version = "0.4.28", features = ["serde"] }
chrono = { version = "0.4.30", features = ["serde"] }
dotenv = "0.15.0"
diesel = { version = "2.0", features = ["postgres", "r2d2", "uuid", "chrono"] }
postgis_diesel = { version = "2.2.1", features = ["serde"] }

23
weather-service/migrations/000001_create_metars/up.sql
View File

@@ -1,24 +1,23 @@
CREATE TABLE IF NOT EXISTS metars (
id SERIAL PRIMARY KEY,
icao TEXT NOT NULL,
id INTEGER PRIMARY KEY GENERATED ALWAYS AS IDENTITY,
raw_text TEXT NOT NULL,
station_id TEXT NOT NULL,
observation_time TEXT NOT NULL,
latitude INTEGER NOT NULL,
longitude INTEGER NOT NULL,
temp_c DOUBLE PRECISION NOT NULL,
dewpoint_c DOUBLE PRECISION NOT NULL,
wind_dir_degrees INTEGER NOT NULL,
wind_speed_kt INTEGER NOT NULL,
visibility_statute_mi TEXT NOT NULL,
altim_in_hg DOUBLE PRECISION NOT NULL,
latitude DOUBLE PRECISION NOT NULL,
longitude DOUBLE PRECISION NOT NULL,
temp_c DOUBLE PRECISION,
dewpoint_c DOUBLE PRECISION,
wind_dir_degrees TEXT,
wind_speed_kt INTEGER,
visibility_statute_mi TEXT,
altim_in_hg DOUBLE PRECISION,
sea_level_pressure_mb DOUBLE PRECISION,
wx_string TEXT,
flight_category TEXT,
three_hr_pressure_tendency_mb DOUBLE PRECISION,
metar_type TEXT,
metar_type TEXT NOT NULL,
max_t_c DOUBLE PRECISION,
min_t_c DOUBLE PRECISION,
precip_in DOUBLE PRECISION,
elevation_m INTEGER
elevation_m INTEGER NOT NULL
);

1
weather-service/src/main.rs
View File

@@ -1,4 +1,3 @@
#[macro_use]
extern crate diesel;
#[macro_use]
extern crate diesel_migrations;

59
weather-service/src/metars/model.rs
View File

@@ -1,7 +1,8 @@
use crate::{error_handler::CustomError, db};
use crate::schema::metars;
use diesel::prelude::*;
use log::{warn, error, debug};
use diesel::{prelude::*, sql_query};
use log::{warn, trace};
use std::collections::HashSet;
use std::io::BufRead;
use quick_xml::{Reader, events::{Event, BytesStart}, Writer, de::Deserializer};
use serde::{Deserialize, Serialize};
@@ -64,7 +65,7 @@ impl Metar {
let mut deserializer = Deserializer::from_str(str);
match Self::deserialize(&mut deserializer) {
Ok(m) => metars.push(m),
Err(err) => warn!("{}", err)
Err(err) => warn!("Error deserializing; {}", err)
};
},
_ => ()
@@ -106,7 +107,8 @@ impl Metar {
}
}
#[derive(Serialize, Deserialize, Queryable)]
#[derive(Serialize, Deserialize, Queryable, QueryableByName)]
#[diesel(table_name = metars)]
pub struct Metars {
pub id: i32,
pub raw_text: String,
@@ -141,15 +143,44 @@ impl Metars {
return Ok(vec![]);
}
let station_icaos: Vec<&str> = icaos.split(',').collect();
let station_query: Vec<String> = station_icaos.iter().map(|icao| format!("'{}'", icao.to_string())).collect();
let mut conn = db::connection()?;
let db_metars: Vec<Metars> = match metars::table
.filter(metars::station_id.eq_any(station_icaos))
.order(metars::id.asc())
.load::<Metars>(&mut conn) {
let mut db_metars: Vec<Metars> = match sql_query(format!("SELECT DISTINCT ON (station_id) * FROM metars WHERE station_id IN ({}) ORDER BY station_id, observation_time DESC", station_query.join(","))).load(&mut conn) {
Ok(m) => m,
Err(err) => return Err(CustomError { error_status_code: 500, error_message: format!("{}", err) })
};
let url = format!("https://beta.aviationweather.gov/cgi-bin/data/metar.php?ids={}&format=xml", icaos);
fn get_missing_metar_icaos(db_metars: &Vec<Metars>, station_icaos: Vec<&str>) -> Vec<String> {
let mut missing_metar_icaos: Vec<String> = vec![];
let current_time = chrono::Local::now().naive_local().timestamp();
let db_metars_set: HashSet<&str> = db_metars.iter().map(|icao| icao.station_id.as_str()).collect();
let station_icaos_set: HashSet<&str> = station_icaos.into_iter().collect();
for difference in db_metars_set.symmetric_difference(&station_icaos_set) {
missing_metar_icaos.push(difference.to_string());
}
for metar in db_metars {
match chrono::NaiveDateTime::parse_and_remainder(&metar.observation_time, "%Y-%m-%dT%H:%M:%S") {
Ok((time, _)) => {
if current_time > (time.timestamp() + 3600) {
trace!("{} METAR data is outdated", metar.station_id);
missing_metar_icaos.push(metar.station_id.to_string());
}
},
Err(err) => {
warn!("Parsing METAR timestamp failed; {}", err);
missing_metar_icaos.push(metar.station_id.to_string());
}
};
}
return missing_metar_icaos;
}
let missing_icaos = get_missing_metar_icaos(&db_metars, station_icaos);
if missing_icaos.is_empty() {
return Ok(db_metars);
}
trace!("Retrieving missing METAR data for {:?}", missing_icaos);
let missing_icaos_string: Vec<String> = missing_icaos.iter().map(|icao| format!("'{}'", icao.to_string())).collect();
let url = format!("https://beta.aviationweather.gov/cgi-bin/data/metar.php?ids={}&format=xml", missing_icaos_string.join(","));
let metars: Vec<Metar> = match reqwest::get(url).await {
Ok(r) => match r.text().await {
Ok(r) => {
@@ -171,15 +202,14 @@ impl Metars {
vec![]
}
};
if metars.len() > 0 {
match diesel::insert_into(metars::table).values(&metars).execute(&mut conn) {
Ok(rows) => debug!("Inserted {} metar rows", rows),
Err(err) => error!("Unable to insert metar data; {}", err)
Ok(rows) => trace!("Inserted {} metar rows", rows),
Err(err) => warn!("Unable to insert metar data; {}", err)
};
}
let mut returned_metars: Vec<Self> = vec![];
for metar in &metars {
// let _ = diesel::insert_into(metars::table)
// .values(metar)
// .execute(&mut conn);
returned_metars.push(Self {
id: 0,
raw_text: metar.raw_text.to_string(),
@@ -213,6 +243,7 @@ impl Metars {
elevation_m: metar.elevation_m,
})
}
returned_metars.append(&mut db_metars);
Ok(returned_metars)
}
}