More pruning, fixed returning metars

This commit is contained in:
2023-09-11 21:10:33 -04:00
parent 75ed374e2f
commit c6ced0299f
3 changed files with 13948 additions and 249748 deletions

View File

@@ -41,6 +41,71 @@ pub struct Metar {
pub elevation_m: i32
}
impl Metar {
pub fn parse(input: String) -> Result<Vec<Self>, CustomError> {
if input.is_empty() {
return Err(CustomError::new(500, "Input is empty".to_string()))
}
let mut reader = Reader::from_str(&input);
let mut buf = Vec::new();
let mut junk_buf: Vec<u8> = Vec::new();
let mut metars: Vec<Self> = vec![];
loop {
match reader.read_event_into(&mut buf) {
Err(e) => panic!("Error at position: {}: {:?}", reader.buffer_position(), e),
Ok(Event::Eof) => break,
Ok(Event::Start(e)) => {
match e.name().as_ref() {
b"METAR" => {
let metar_bytes = Self::read_to_end_into_buffer(&mut reader, &e, &mut junk_buf).unwrap();
let str = std::str::from_utf8(&metar_bytes).unwrap();
let mut deserializer = Deserializer::from_str(str);
match Self::deserialize(&mut deserializer) {
Ok(m) => metars.push(m),
Err(err) => warn!("{}", err)
};
},
_ => ()
}
},
_ => ()
}
}
return Ok(metars)
}
// https://capnfabs.net/posts/parsing-huge-xml-quickxml-rust-serde/
pub fn read_to_end_into_buffer<R: BufRead>(reader: &mut Reader<R>, start_tag: &BytesStart, junk_buf: &mut Vec<u8>) -> Result<Vec<u8>, quick_xml::Error> {
let mut depth = 0;
let mut output_buf: Vec<u8> = Vec::new();
let mut w = Writer::new(&mut output_buf);
let tag_name = start_tag.name();
w.write_event(Event::Start(start_tag.clone()))?;
loop {
junk_buf.clear();
let event = reader.read_event_into(junk_buf)?;
w.write_event(&event)?;
match event {
Event::Start(e) if e.name() == tag_name => depth += 1,
Event::End(e) if e.name() == tag_name => {
if depth == 0 {
return Ok(output_buf);
}
depth -= 1;
}
Event::Eof => {
panic!("EOF")
}
_ => {}
}
}
}
}
#[derive(Serialize, Deserialize, Queryable)]
pub struct Metars {
pub id: i32,
@@ -85,10 +150,10 @@ impl Metars {
Err(err) => return Err(CustomError { error_status_code: 500, error_message: format!("{}", err) })
};
let url = format!("https://beta.aviationweather.gov/cgi-bin/data/metar.php?ids={}&format=xml", icaos);
let metars: Vec<Metars> = match reqwest::get(url).await {
let metars: Vec<Metar> = match reqwest::get(url).await {
Ok(r) => match r.text().await {
Ok(r) => {
match Metars::parse(r) {
match Metar::parse(r) {
Ok(m) => m,
Err(err) => {
warn!("{}", err);
@@ -106,9 +171,13 @@ impl Metars {
vec![]
}
};
let mut returned_metars: Vec<Self> = vec![];
for metar in &metars {
let _ = diesel::insert_into(metars::table)
.values(Metar {
.values(metar)
.execute(&mut conn);
returned_metars.push(Self {
id: 0,
raw_text: metar.raw_text.to_string(),
station_id: metar.station_id.to_string(),
observation_time: metar.observation_time.to_string(),
@@ -139,71 +208,7 @@ impl Metars {
precip_in: metar.precip_in,
elevation_m: metar.elevation_m,
})
.execute(&mut conn);
}
Ok(metars)
}
pub fn parse(input: String) -> Result<Vec<Self>, CustomError> {
if input.is_empty() {
return Err(CustomError::new(500, "Input is empty".to_string()))
}
let mut reader = Reader::from_str(&input);
let mut buf = Vec::new();
let mut junk_buf: Vec<u8> = Vec::new();
let mut metars: Vec<Self> = vec![];
loop {
match reader.read_event_into(&mut buf) {
Err(e) => panic!("Error at position: {}: {:?}", reader.buffer_position(), e),
Ok(Event::Eof) => break,
Ok(Event::Start(e)) => {
match e.name().as_ref() {
b"METAR" => {
let metar_bytes = Metars::read_to_end_into_buffer(&mut reader, &e, &mut junk_buf).unwrap();
let str = std::str::from_utf8(&metar_bytes).unwrap();
let mut deserializer = Deserializer::from_str(str);
match Metars::deserialize(&mut deserializer) {
Ok(m) => metars.push(m),
Err(err) => warn!("{}", err)
};
},
_ => ()
}
},
_ => ()
}
}
return Ok(metars)
}
// https://capnfabs.net/posts/parsing-huge-xml-quickxml-rust-serde/
pub fn read_to_end_into_buffer<R: BufRead>(reader: &mut Reader<R>, start_tag: &BytesStart, junk_buf: &mut Vec<u8>) -> Result<Vec<u8>, quick_xml::Error> {
let mut depth = 0;
let mut output_buf: Vec<u8> = Vec::new();
let mut w = Writer::new(&mut output_buf);
let tag_name = start_tag.name();
w.write_event(Event::Start(start_tag.clone()))?;
loop {
junk_buf.clear();
let event = reader.read_event_into(junk_buf)?;
w.write_event(&event)?;
match event {
Event::Start(e) if e.name() == tag_name => depth += 1,
Event::End(e) if e.name() == tag_name => {
if depth == 0 {
return Ok(output_buf);
}
depth -= 1;
}
Event::Eof => {
panic!("EOF")
}
_ => {}
}
}
Ok(returned_metars)
}
}