bsherriff/siren
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added lib to service

Browse Source
This commit is contained in:
Benjamin Sherriff
2023-10-04 19:58:54 -04:00
parent cee9dbdc81
commit ecc65222b6
20 changed files with 169 additions and 166 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
bot/Cargo.toml
View File

@@ -13,6 +13,7 @@ dotenv = "0.15.0"
serde_json = "1.0.107"
log = "0.4.20"
env_logger = "0.10.0"
service = { path = "../service" }
[dependencies.serenity]
version = "0.11.6"

37
bot/Dockerfile Normal file
View File

@@ -0,0 +1,37 @@
# Builder
FROM rust:1.72.1-bookworm as builder
WORKDIR /builder
COPY src ./src
COPY Cargo.toml ./
RUN cargo build --release
# Packages
FROM debian:bullseye-slim as packages
WORKDIR /packages
RUN apt-get update && apt-get install -y curl tar xz-utils && \
curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux > yt-dlp && \
chmod +x yt-dlp && \
curl -L https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz > ffmpeg.tar.xz && \
tar -xJf ffmpeg.tar.xz --wildcards */bin/ffmpeg --transform='s/^.*\///' && rm ffmpeg.tar.xz
# FROM debian:bullseye-slim as libraries
# WORKDIR /libraries
# RUN apt-get update && apt-get install -y unzip && \
# curl -L https://download.pytorch.org/libtorch/cu117/libtorch-cxx11-abi-shared-with-deps-2.0.1%2Bcu117.zip > libtorch.zip && \
# unzip libtorch.zip && rm libtorch.zip
# Runner
FROM debian:bullseye-slim as runtime
WORKDIR /bot
RUN apt-get update && apt-get install -y libopus-dev libpq5 libpq-dev && apt-get auto-remove -y
COPY --from=builder /builder/target/release/bot /usr/local/bin/bot
COPY --from=packages /packages /usr/bin
# COPY --from=libraries /libraries /usr/lib
# ARG LIBTORCH=/usr/lib/libtorch
# ARG LD_LIBRARY_PATH=${LIBTORCH}/lib:${LD_LIBRARY_PATH}
# ADD migrations ./
CMD ["bot"]

27
bot/Makefile Normal file
View File

@@ -0,0 +1,27 @@
#!make
SHELL := /bin/bash
include .env
.PHONY: help build test up down exec clean
help: ## Help command
@echo
@cat Makefile | grep -E '^[a-zA-Z\/_-]+:.*?## .*$$' | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'
@echo
build: ## Build the docker image
docker compose build
db: ## Start the docker database
docker compose up -d db
up: ## Start the app
docker compose up -d
down: ## Stop the app
docker compose down
clean:
docker compose down && \
docker image rm siren-bot

20
bot/docker-compose.yml Normal file
View File

@@ -0,0 +1,20 @@
version: '3.8'
services:
bot:
image: siren-bot:${BOT_VERSION:-latest}
container_name: siren-bot
build:
context: .
dockerfile: ./Dockerfile
args:
- VERSION=${BOT_VERSION:-latest}
env_file:
- .env
networks:
- frontend
restart: unless-stopped
networks:
frontend:

60
bot/src/commands/oai.rs
View File

@@ -6,8 +6,7 @@ use serenity::model::Permissions;
use serenity::model::channel::Message;
use serenity::model::prelude::{ChannelType, PermissionOverwrite, PermissionOverwriteType};
use serenity::prelude::*;
use crate::error_handler::BotError;
use siren::{GetResponse, ServiceError};
pub struct OAI {
pub client: reqwest::Client,
@@ -126,51 +125,8 @@ enum ResponseEvent {
ResponseError(ResponseError)
}
#[derive(Serialize, Deserialize)]
pub struct GetResponse<T> {
pub data: T,
#[serde(skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>
}
#[derive(Serialize, Deserialize)]
pub struct Metadata {
pub total: i32,
pub limit: i32,
pub page: i32,
pub pages: i32
}
#[derive(Serialize, Deserialize)]
pub struct QueryMessage {
pub id: String,
pub guild_id: i64,
pub channel_id: i64,
pub user_id: i64,
pub created: i64,
pub model: String,
pub request: String,
pub response: String,
pub request_tags: Vec<String>,
pub response_tags: Vec<String>,
}
#[derive(Serialize, Deserialize)]
pub struct InsertMessage {
pub id: String,
pub guild_id: i64,
pub channel_id: i64,
pub user_id: i64,
pub created: i64,
pub model: String,
pub request: String,
pub response: String,
pub request_tags: Vec<String>,
pub response_tags: Vec<String>,
}
impl OAI {
async fn get_request(&self, request: ChatCompletionRequest) -> Result<ChatCompletionResponse, BotError> {
async fn get_request(&self, request: ChatCompletionRequest) -> Result<ChatCompletionResponse, ServiceError> {
let uri = format!("{}/chat/completions", self.base_url);
let body = serde_json::to_string(&request).unwrap();
trace!("Sending request to {}: {}", uri, body);
@@ -204,7 +160,7 @@ impl OAI {
Ok(response)
}
async fn get_messages(&self, guild_id: u64, channel_id: u64, author_id: u64) -> Result<GetResponse<Vec<QueryMessage>>, BotError> {
async fn get_messages(&self, guild_id: u64, channel_id: u64, author_id: u64) -> Result<GetResponse<Vec<siren::Message>>, ServiceError> {
let uri = format!("{}/messages?guild_id={}&channel_id={}&author_id={}&limit={}", self.service_url, guild_id, channel_id, author_id, self.max_context_questions);
let value = self.client
.get(&uri)
@@ -213,23 +169,23 @@ impl OAI {
.json::<Value>()
.await?;
let response = serde_json::from_value::<GetResponse<Vec<QueryMessage>>>(value)?;
let response = serde_json::from_value::<GetResponse<Vec<siren::Message>>>(value)?;
Ok(response)
}
async fn store_message(&self, message: InsertMessage) -> Result<QueryMessage, BotError> {
async fn store_message(&self, message: siren::Message) -> Result<siren::Message, ServiceError> {
let uri = format!("{}/messages", self.service_url);
trace!("Sending request to {}", uri);
let value = self.client
.post(&uri)
.json::<InsertMessage>(&message)
.json::<siren::Message>(&message)
.send()
.await?
.json::<Value>()
.await?;
trace!("Received response from Service: {:?}", value);
let response = serde_json::from_value::<QueryMessage>(value)?;
let response = serde_json::from_value::<siren::Message>(value)?;
Ok(response)
}
}
@@ -314,7 +270,7 @@ pub async fn generate_response(ctx: &Context, msg: &Message, oai: &OAI) {
debug!("Processing response received from OpenAI");
if !r.choices.is_empty() {
let res = r.choices[0].message.content.clone();
if let Err(err) = oai.store_message(InsertMessage {
if let Err(err) = oai.store_message(siren::Message {
id: r.id,
guild_id: guild_id.0 as i64,
channel_id: response_channel.0 as i64,

35
bot/src/error_handler.rs
View File

@@ -1,35 +0,0 @@
use serde::{Deserialize, Serialize};
use std::fmt;
#[derive(Debug, Deserialize, Serialize)]
pub struct BotError {
pub status: u16,
pub message: String,
}
impl BotError {
pub fn new(error_status_code: u16, error_message: String) -> BotError {
BotError {
status: error_status_code,
message: error_message,
}
}
}
impl fmt::Display for BotError {
fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
f.write_str(self.message.as_str())
}
}
impl From<reqwest::Error> for BotError {
fn from(error: reqwest::Error) -> BotError {
BotError::new(500, format!("Unknown reqwest error: {}", error))
}
}
impl From<serde_json::Error> for BotError {
fn from(error: serde_json::Error) -> BotError {
BotError::new(500, format!("Unknown serde_json error: {}", error))
}
}

1
bot/src/main.rs
View File

@@ -18,7 +18,6 @@ use songbird::SerenityInit;
use crate::commands::oai::GPTModel;
mod commands;
mod error_handler;
struct Handler {
// Open AI Config

2
service/.env.TEMPLATE
View File

@@ -1,4 +1,4 @@
RUST_LOG=warn,siren=info
RUST_LOG=warn,service=info
COMPOSE_PROJECT_NAME=siren
DATABASE_USER=siren

6
service/Cargo.toml
View File

@@ -1,5 +1,5 @@
[package]
name = "siren"
name = "service"
version = "0.2.4"
edition = "2021"
authors = ["Ben Sherriff <hello@bensherriff.com>"]
@@ -7,6 +7,10 @@ repository = "https://github.com/bensherriff/siren"
readme = "README.md"
license = "GPL-3.0-or-later"
[lib]
name = "siren"
path = "src/lib.rs"
[dependencies]
actix-web = "4.4.0"
actix-rt = "2.9.0"

40
service/Dockerfile
View File

@@ -1,36 +1,12 @@
# Builder
FROM rust:1.72.1-bookworm as builder
WORKDIR /siren
ADD src ./src/
ADD Cargo.toml ./
RUN apt-get update && apt-get install -y cmake && \
cargo build --release --bin siren
# Packages
FROM debian:bullseye-slim as packages
WORKDIR /packages
RUN apt-get update && apt-get install -y curl tar xz-utils && \
curl -L https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp_linux > yt-dlp && \
chmod +x yt-dlp && \
curl -L https://github.com/yt-dlp/FFmpeg-Builds/releases/download/latest/ffmpeg-master-latest-linux64-gpl.tar.xz > ffmpeg.tar.xz && \
tar -xJf ffmpeg.tar.xz --wildcards */bin/ffmpeg --transform='s/^.*\///' && rm ffmpeg.tar.xz
WORKDIR /service
USER root
# FROM debian:bullseye-slim as libraries
# WORKDIR /libraries
# RUN apt-get update && apt-get install -y unzip && \
# curl -L https://download.pytorch.org/libtorch/cu117/libtorch-cxx11-abi-shared-with-deps-2.0.1%2Bcu117.zip > libtorch.zip && \
# unzip libtorch.zip && rm libtorch.zip
COPY migrations ./migrations
COPY data ./data
COPY src ./src
COPY Cargo.toml ./
# Runner
FROM debian:bullseye-slim as runtime
WORKDIR /siren
RUN apt-get update && apt-get install -y libopus-dev libpq5 libpq-dev && apt-get auto-remove -y
COPY --from=builder /siren/target/release/siren /usr/local/bin/siren
COPY --from=packages /packages /usr/bin
# COPY --from=libraries /libraries /usr/lib
# ARG LIBTORCH=/usr/lib/libtorch
# ARG LD_LIBRARY_PATH=${LIBTORCH}/lib:${LD_LIBRARY_PATH}
# ADD migrations ./
CMD ["siren"]
RUN cargo build --release
CMD ["./target/release/service"]

20
service/Makefile
View File

@@ -2,11 +2,6 @@
SHELL := /bin/bash
include .env
include .version
export $(shell sed 's/=.*//' .env)
export $(shell sed 's/=.*//' .version)
SIREN_IMAGES = $(shell docker images 'siren' -a -q)
.PHONY: help build test up down exec clean
@@ -16,10 +11,7 @@ help: ## Help command
@echo
build: ## Build the docker image
docker build -t siren:${SIREN_VERSION} .
test: ## Run the docker app as a container
docker run --env-file .env -it --rm --name siren siren:${SIREN_VERSION}
docker compose build
db: ## Start the docker database
docker compose up -d db
@@ -30,8 +22,8 @@ up: ## Start the app
down: ## Stop the app
docker compose down
exec: ## Enter running docker container
docker exec -it siren bash
clean: ## Cleanup docker images
docker rmi $(SIREN_IMAGES)
clean:
docker compose down && \
docker image rm siren-service || \
docker network rm siren_frontend || \
docker network rm siren-backend

4
service/build.rs
View File

@@ -1,4 +0,0 @@
fn main() {
let home = std::env::var("HOME").expect("${HOME} is missing");
println!("cargo:rustc-env=LD_LIBRARY_PATH={home}/.pyenv/versions/3.11.4/lib");
}

6
service/docker-compose.yml
View File

@@ -1,8 +1,8 @@
version: '3.8'
services:
siren:
image: siren:${SIREN_VERSION:-latest}
service:
image: siren-service:${SIREN_VERSION:-latest}
container_name: siren-service
build:
context: .
@@ -16,6 +16,8 @@ services:
DATABASE_PORT: 5432
SERVICE_HOST: siren
SERVICE_PORT: 5000
ports:
- ${SERVICE_PORT:-5000}:5000
depends_on:
- db
networks:

3
service/src/db/messages/model.rs
View File

@@ -1,7 +1,8 @@
use diesel::prelude::*;
use serde::{Deserialize, Serialize};
use siren::ServiceError;
use crate::{db::schema::messages::{self}, error_handler::ServiceError};
use crate::db::schema::messages::{self};
#[derive(Queryable, Selectable, Serialize, Deserialize)]
#[diesel(table_name = messages)]

3
service/src/db/messages/routes.rs
View File

@@ -1,8 +1,9 @@
use actix_web::{get, post, web, HttpResponse, HttpRequest, ResponseError};
use log::error;
use serde::{Serialize, Deserialize};
use siren::{GetResponse, Metadata, ServiceError};
use crate::{db::{messages::{QueryMessage, QueryFilters, InsertMessage}, GetResponse, Metadata}, error_handler::ServiceError};
use crate::db::messages::{QueryMessage, QueryFilters, InsertMessage};
#[derive(Serialize, Deserialize)]
struct GetAllParams {

18
service/src/db/mod.rs
View File

@@ -1,6 +1,5 @@
use crate::error_handler::ServiceError;
use diesel::{r2d2::ConnectionManager, PgConnection};
use serde::{Deserialize, Serialize};
use siren::ServiceError;
use crate::diesel_migrations::MigrationHarness;
use lazy_static::lazy_static;
use log::{error, info};
@@ -55,18 +54,3 @@ pub fn connection() -> Result<DbConnection, ServiceError> {
pub fn load_data() {
spells::load_data();
}
#[derive(Serialize, Deserialize)]
pub struct GetResponse<T> {
pub data: T,
#[serde(skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>
}
#[derive(Serialize, Deserialize)]
pub struct Metadata {
pub total: i32,
pub limit: i32,
pub page: i32,
pub pages: i32
}

3
service/src/db/spells/model.rs
View File

@@ -1,7 +1,8 @@
use diesel::prelude::*;
use serde::{Deserialize, Serialize};
use siren::ServiceError;
use crate::{db::{schema::spells::{self}, classes::AbilityType, conditions::ConditionType}, error_handler::ServiceError};
use crate::db::{schema::spells::{self}, classes::AbilityType, conditions::ConditionType};
use super::{SchoolType, CastingTime, CastingType, SpellAttackType, SpellDamageType, Range, Area, Components, Duration, Source, Description, DurationType};

3
service/src/db/spells/routes.rs
View File

@@ -1,8 +1,9 @@
use actix_web::{get, post, put, delete, web, HttpResponse, HttpRequest, ResponseError};
use log::error;
use serde::{Serialize, Deserialize};
use siren::{GetResponse, Metadata, ServiceError};
use crate::{db::{spells::{QuerySpell, QueryFilters}, GetResponse, Metadata}, error_handler::ServiceError};
use crate::db::spells::{QuerySpell, QueryFilters};
use super::{Spell, InsertSpell};

45
service/src/error_handler.rs → service/src/lib.rs
View File

@@ -1,9 +1,38 @@
use actix_web::{ResponseError, HttpResponse};
use diesel::result::Error as DieselError;
use reqwest::StatusCode;
use serde::{Deserialize, Serialize};
use serde::{Serialize, Deserialize};
use std::fmt;
#[derive(Serialize, Deserialize)]
pub struct Message {
pub id: String,
pub guild_id: i64,
pub channel_id: i64,
pub user_id: i64,
pub created: i64,
pub model: String,
pub request: String,
pub response: String,
pub request_tags: Vec<String>,
pub response_tags: Vec<String>,
}
#[derive(Serialize, Deserialize)]
pub struct GetResponse<T> {
pub data: T,
#[serde(skip_serializing_if = "Option::is_none")]
pub metadata: Option<Metadata>
}
#[derive(Serialize, Deserialize)]
pub struct Metadata {
pub total: i32,
pub limit: i32,
pub page: i32,
pub pages: i32
}
#[derive(Debug, Deserialize, Serialize)]
pub struct ServiceError {
pub status: u16,
@@ -40,6 +69,18 @@ impl From<DieselError> for ServiceError {
}
}
impl From<reqwest::Error> for ServiceError {
fn from(error: reqwest::Error) -> ServiceError {
ServiceError::new(500, format!("Unknown reqwest error: {}", error))
}
}
impl From<serde_json::Error> for ServiceError {
fn from(error: serde_json::Error) -> ServiceError {
ServiceError::new(500, format!("Unknown serde_json error: {}", error))
}
}
impl ResponseError for ServiceError {
fn error_response(&self) -> HttpResponse {
let status_code = match StatusCode::from_u16(self.status) {
@@ -54,4 +95,4 @@ impl ResponseError for ServiceError {
HttpResponse::build(status_code).json(serde_json::json!({ "status": status_code.as_u16(), "message": error_message }))
}
}
}

1
service/src/main.rs
View File

@@ -9,7 +9,6 @@ use actix_web::{HttpServer, App};
use dotenv::dotenv;
use log::{error, info};
mod error_handler;
mod db;
#[actix_web::main]