From 8afc98ed3330a057be8da7738035fcdb4b88913d Mon Sep 17 00:00:00 2001 From: Ben Sherriff Date: Mon, 18 Dec 2023 18:34:14 -0500 Subject: [PATCH] Fixed import --- service/Cargo.lock | 25 ++++----- service/Cargo.toml | 1 + service/docker-compose.yml | 1 + service/src/airports/model.rs | 13 +++++ service/src/airports/routes.rs | 54 +++++++++++++++---- service/src/auth/routes.rs | 35 ++++++++++++ service/src/scheduler.rs | 7 ++- ui/src/api/airport.ts | 10 ++-- ui/src/components/Admin/AirportTablePanel.tsx | 42 ++++++++++++--- 9 files changed, 156 insertions(+), 32 deletions(-) diff --git a/service/Cargo.lock b/service/Cargo.lock index 09ca081..8afaddb 100644 --- a/service/Cargo.lock +++ b/service/Cargo.lock @@ -793,9 +793,9 @@ dependencies = [ [[package]] name = "futures-channel" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +checksum = "ff4dd66668b557604244583e3e1e1eada8c5c2e96a6d0d6653ede395b78bbacb" dependencies = [ "futures-core", "futures-sink", @@ -803,9 +803,9 @@ dependencies = [ [[package]] name = "futures-core" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" +checksum = "eb1d22c66e66d9d72e1758f0bd7d4fd0bee04cad842ee34587d68c07e45d088c" [[package]] name = "futures-executor" @@ -826,9 +826,9 @@ checksum = "8bf34a163b5c4c52d0478a4d757da8fb65cabef42ba90515efee0f6f9fa45aaa" [[package]] name = "futures-macro" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +checksum = "53b153fd91e4b0147f4aced87be237c98248656bb01050b96bf3ee89220a8ddb" dependencies = [ "proc-macro2", "quote", @@ -837,21 +837,21 @@ dependencies = [ [[package]] name = "futures-sink" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" +checksum = "e36d3378ee38c2a36ad710c5d30c2911d752cb941c00c72dbabfb786a7970817" [[package]] name = "futures-task" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" +checksum = "efd193069b0ddadc69c46389b740bbccdd97203899b48d09c5f7969591d6bae2" [[package]] name = "futures-util" -version = "0.3.28" +version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +checksum = "a19526d624e703a3179b3d322efec918b6246ea0fa51d41124525f00f1cc8104" dependencies = [ "futures-channel", "futures-core", @@ -1801,6 +1801,7 @@ dependencies = [ "diesel_migrations", "dotenv", "env_logger", + "futures-util", "jsonwebtoken", "lazy_static", "log", diff --git a/service/Cargo.toml b/service/Cargo.toml index 79f64bb..057c305 100644 --- a/service/Cargo.toml +++ b/service/Cargo.toml @@ -33,3 +33,4 @@ jsonwebtoken = "9.0.0" redis = { version = "0.23.3", features = ["tokio-comp", "connection-manager", "r2d2"] } rustix = "0.38.19" # https://github.com/imsnif/bandwhich/issues/284 regex = "1.10.2" +futures-util = "0.3.29" diff --git a/service/docker-compose.yml b/service/docker-compose.yml index 9615867..eee96cf 100644 --- a/service/docker-compose.yml +++ b/service/docker-compose.yml @@ -54,6 +54,7 @@ services: REDIS_PORT: 6379 SERVICE_HOST: service SERVICE_PORT: 5000 + KEYS_DIR_PATH: /keys volumes: - ${KEYS_DIR_PATH}:/keys ports: diff --git a/service/src/airports/model.rs b/service/src/airports/model.rs index ca89af3..de5e0b9 100644 --- a/service/src/airports/model.rs +++ b/service/src/airports/model.rs @@ -248,6 +248,19 @@ impl QueryAirport { Ok(airport) } + pub fn insert_all (airports: Vec) -> Result, ServiceError> { + let mut conn: r2d2::PooledConnection> = db::connection()?; + let mut inserted_airports: Vec = vec![]; + for airport in airports { + let airport = Self::from(airport); + let airport = diesel::insert_into(airports::table) + .values(airport) + .get_result(&mut conn)?; + inserted_airports.push(airport); + } + Ok(inserted_airports) + } + pub fn update(icao: String, airport: Self) -> Result { let mut conn = db::connection()?; let airport = diesel::update(airports::table) diff --git a/service/src/airports/routes.rs b/service/src/airports/routes.rs index df0adfb..8443c88 100644 --- a/service/src/airports/routes.rs +++ b/service/src/airports/routes.rs @@ -1,6 +1,8 @@ use std::str::FromStr; +use futures_util::stream::StreamExt as _; -use crate::{airports::{QueryAirport, QueryFilters, QueryOrderField, QueryOrderBy, Airport}, db::{self, Response, Metadata}, auth::{JwtAuth, verify_role}}; +use crate::{airports::{QueryAirport, QueryFilters, QueryOrderField, QueryOrderBy, Airport}, db::{Response, Metadata}, auth::{JwtAuth, verify_role}}; +use actix_multipart::Multipart; use actix_web::{delete, get, post, put, web, HttpResponse, HttpRequest, ResponseError}; use log::{error, warn}; use postgis_diesel::types::{Polygon, Point}; @@ -18,16 +20,48 @@ struct GetAllParameters { } #[post("/import")] -async fn import(auth: JwtAuth) -> HttpResponse { - let _ = match verify_role(&auth, "admin") { - Ok(_) => {}, - Err(err) => return ResponseError::error_response(&err) +async fn import(mut payload: Multipart, auth: JwtAuth) -> HttpResponse { + if let Err(err) = verify_role(&auth, "admin") { + return ResponseError::error_response(&err) }; - let count = db::import_data(); - HttpResponse::Ok().json(Response { - data: count, - meta: None - }) + + + while let Some(item) = payload.next().await { + let mut bytes = web::BytesMut::new(); + let mut field = match item { + Ok(field) => field, + Err(err) => return ResponseError::error_response(&err) + }; + + // Build bytes from chunks + while let Some(chunk) = field.next().await { + let data = match chunk { + Ok(data) => data, + Err(err) => { + error!("Failed to get chunk: {}", err); + return ResponseError::error_response(&err); + } + }; + bytes.extend_from_slice(&data); + } + + // Convert bytes to Vec + let airports: Vec = match serde_json::from_slice(&bytes) { + Ok(a) => a, + Err(err) => { + error!("Failed to parse JSON: {}", err); + return ResponseError::error_response(&err); + } + }; + + // Convert Vec to Vec and insert into database + let query_airports: Vec = airports.into_iter().map(|a| a.into()).collect(); + match QueryAirport::insert_all(query_airports) { + Ok(_) => {}, + Err(err) => return ResponseError::error_response(&err) + }; + }; + HttpResponse::Ok().finish() } #[get("")] diff --git a/service/src/auth/routes.rs b/service/src/auth/routes.rs index 03630ec..679b203 100644 --- a/service/src/auth/routes.rs +++ b/service/src/auth/routes.rs @@ -127,6 +127,41 @@ async fn login(request: web::Json) -> HttpResponse { } } +#[get("/session")] +async fn session(req: HttpRequest) -> HttpResponse { + let keys_dir = env::var("KEYS_DIR_PATH").expect("KEYS_DIR_PATH must be set"); + // If there is a access_token cookie, check if it is valid + let has_session = match req.cookie("access_token") { + Some(cookie) => { + let access_token = cookie.value().to_string(); + let public_key = std::fs::read_to_string(format!("{}access_public_key.pem", keys_dir)) + .expect("Unable to read refresh public key"); + match verify_token(&access_token, &public_key) { + Ok(_) => true, + Err(_) => false + } + }, + None => false + }; + if !has_session { + // If there is a refresh_token cookie, check if it is valid + match req.cookie("refresh_token") { + Some(cookie) => { + let refresh_token = cookie.value().to_string(); + let public_key = std::fs::read_to_string(format!("{}/refresh_public_key.pem", keys_dir)) + .expect("Unable to read refresh public key"); + match verify_token(&refresh_token, &public_key) { + Ok(_) => return HttpResponse::Ok().json(true), + Err(_) => return HttpResponse::Ok().json(false) + }; + }, + None => return HttpResponse::Ok().json(false) + }; + } else { + return HttpResponse::Ok().json(true) + } +} + #[derive(Serialize, Deserialize)] struct RefreshParams { refresh_token_rotation: Option diff --git a/service/src/scheduler.rs b/service/src/scheduler.rs index f7d732a..2ccfb36 100644 --- a/service/src/scheduler.rs +++ b/service/src/scheduler.rs @@ -35,6 +35,11 @@ pub fn update_airports() { let mut peekable = airport_icaos.into_iter().peekable(); let mut observation_time = chrono::Utc::now().timestamp(); + if peekable.peek().is_none() { + sleep(Duration::from_secs(3600)).await; + continue; + } + while peekable.peek().is_some() { let chunk: Vec = peekable.by_ref().take(limit as usize).collect(); let icao_string = chunk.join(","); @@ -58,7 +63,7 @@ pub fn update_airports() { // Sleep until the earliest observation time is 1 hour old // Bounded by 1 and 3600 seconds let now = chrono::Utc::now().timestamp(); - let sleep_time = std::cmp::min(std::cmp::max(1, now - (observation_time + (3600))), 3600); + let sleep_time = std::cmp::min(std::cmp::max(1, now - (observation_time + 3600)), 3600); debug!("Next update in {} seconds", sleep_time); sleep(Duration::from_secs(sleep_time as u64)).await; } diff --git a/ui/src/api/airport.ts b/ui/src/api/airport.ts index 992cf5d..f66eaa5 100644 --- a/ui/src/api/airport.ts +++ b/ui/src/api/airport.ts @@ -63,7 +63,11 @@ export async function updateAirport({ airport }: { airport: Airport }): Promise< return response?.json() || { data: undefined }; } -export async function importAirports(): Promise { - const response = await postRequest('airports/import'); - return response?.json() || { data: undefined }; +export async function importAirports(payload: File): Promise { + const data = new FormData(); + data.append('data', payload); + const response = await postRequest('airports/import', data, { + type: 'form' + }); + return response?.status == 200; } diff --git a/ui/src/components/Admin/AirportTablePanel.tsx b/ui/src/components/Admin/AirportTablePanel.tsx index 0915516..5d56952 100644 --- a/ui/src/components/Admin/AirportTablePanel.tsx +++ b/ui/src/components/Admin/AirportTablePanel.tsx @@ -1,6 +1,6 @@ import { getAirports, importAirports, removeAirport } from "@/api/airport"; import { Airport, airportCategoryToText } from "@/api/airport.types"; -import { Text, Button, Card, Group, Pagination, Table, TextInput, rem, UnstyledButton, Center, Flex, Container, Grid, Space } from "@mantine/core"; +import { Text, Button, Card, Group, Pagination, Table, TextInput, rem, UnstyledButton, Center, Flex, Container, Grid, Space, FileButton } from "@mantine/core"; import { HiChevronUp, HiChevronDown, HiSelector } from "react-icons/hi"; import { useEffect, useState } from "react"; import { CiSearch } from "react-icons/ci"; @@ -81,12 +81,14 @@ export default function AirportTablePanel({ setAirport }: { setAirport: (airport - { - await importAirports(); - await getAirportData(); + { + if (payload instanceof File) { + await importAirports(payload); + await getAirportData(); + } }}> Import - + { @@ -102,7 +104,35 @@ export default function AirportTablePanel({ setAirport }: { setAirport: (airport } -function PanelButton({ children, color = 'blue', onClick }: {children: any, color?: string, onClick: () => Promise }) { +interface PanelButtonProps { + children: any; + color?: string; + onClick?: () => Promise; +} + +interface PanelFileButtonProps { + children: any; + color?: string; + multiple?: boolean; + accept?: string; + onChange?: (payload: File|File[]|null) => Promise; +} + +function PanelFileButton({ children, multiple = false, accept, color, onChange = async () => {} }: PanelFileButtonProps) { + const [loading, setLoading] = useState(false); + return { + setLoading(true); + onChange(e).then(() => setLoading(false)); + }} + > + {(props) => } + +} + +function PanelButton({ children, color = 'blue', onClick = async () => {} }: PanelButtonProps) { const [loading, setLoading] = useState(false); return