From 8f58c573abf8a3ae15b4f04a4d72585406bb25e2 Mon Sep 17 00:00:00 2001 From: slonkazoid Date: Mon, 16 Dec 2024 01:49:04 +0300 Subject: [PATCH] add blagging support --- CONFIG.md | 12 ++- Cargo.lock | 91 +++++++++++++++---- Cargo.toml | 2 + src/app.rs | 10 +- src/config.rs | 24 +++++ src/error.rs | 46 +++++----- src/main.rs | 67 +++++++------- src/post/blag.rs | 182 +++++++++++++++++++++++++++++++++++++ src/post/markdown_posts.rs | 8 ++ src/post/mod.rs | 16 +++- templates/footer.hbs | 4 +- 11 files changed, 375 insertions(+), 87 deletions(-) create mode 100644 src/post/blag.rs diff --git a/CONFIG.md b/CONFIG.md index 551a621..8cc790b 100644 --- a/CONFIG.md +++ b/CONFIG.md @@ -5,10 +5,13 @@ the configuration format, with defaults, is documented below: ```toml title = "bingus-blog" # title of the blog # description of the blog -description = "blazingly fast markdown blog software written in rust memory safe" -markdown_access = true # allow users to see the raw markdown of a post - # endpoint: /posts/.md +description = "blazingly fast blog software written in rust memory safe" +raw_access = true # allow users to see the raw source of a post js_enable = true # enable javascript (required for sorting and dates) +engine = "markdown" # choose which post engine to use + # options: "markdown", "blag" + # absolutely do not use "blag" unless you know exactly + # what you are getting yourself into. [style] date_format = "RFC3339" # format string used to format dates in the backend @@ -52,6 +55,9 @@ compression_level = 3 # zstd compression level, 3 is recommended syntect.load_defaults = false # include default syntect themes syntect.themes_dir = "themes" # directory to include themes from syntect.theme = "Catppuccin Mocha" # theme file name (without `.tmTheme`) + +[blag] +bin = "blag" # path to blag binary ``` configuration is done in [TOML](https://toml.io/) diff --git a/Cargo.lock b/Cargo.lock index ef150b1..7feb129 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -310,6 +310,7 @@ dependencies = [ "console-subscriber", "derive_more", "fronma", + "futures", "handlebars", "include_dir", "mime_guess", @@ -849,42 +850,92 @@ dependencies = [ ] [[package]] -name = "futures-channel" -version = "0.3.30" +name = "futures" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", + "futures-sink", ] [[package]] name = "futures-core" -version = "0.3.30" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" +checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" [[package]] -name = "futures-sink" -version = "0.3.30" +name = "futures-executor" +version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" - -[[package]] -name = "futures-task" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" - -[[package]] -name = "futures-util" -version = "0.3.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" dependencies = [ "futures-core", "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.73", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + +[[package]] +name = "futures-task" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" + +[[package]] +name = "futures-util" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 0ff9514..3d1f7f0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,6 +39,7 @@ comrak = { version = "0.22.0", features = [ console-subscriber = { version = "0.2.0", optional = true } derive_more = "0.99.17" fronma = "0.2.0" +futures = "0.3.31" handlebars = "6.0.0" include_dir = "0.7.4" mime_guess = "2.0.5" @@ -54,6 +55,7 @@ tokio = { version = "1.37.0", features = [ "macros", "rt-multi-thread", "signal", + "process", ] } tokio-util = { version = "0.7.10", default-features = false } toml = "0.8.12" diff --git a/src/app.rs b/src/app.rs index d8ae6e8..772b8db 100644 --- a/src/app.rs +++ b/src/app.rs @@ -66,11 +66,11 @@ struct PostTemplate<'a> { meta: &'a PostMetadata, rendered: String, rendered_in: RenderStats, - markdown_access: bool, js: bool, color: Option<&'a str>, joined_tags: String, style: &'a StyleConfig, + raw_name: Option<&'a str>, } #[derive(Deserialize)] @@ -240,6 +240,7 @@ async fn post( let joined_tags = meta.tags.join(", "); let reg = reg.read().await; + let raw_name; let rendered = reg.render( "post", &PostTemplate { @@ -247,7 +248,6 @@ async fn post( meta, rendered, rendered_in, - markdown_access: config.markdown_access, js: config.js_enable, color: meta .color @@ -255,6 +255,12 @@ async fn post( .or(config.style.default_color.as_deref()), joined_tags, style: &config.style, + raw_name: if config.markdown_access { + raw_name = posts.get_raw(&meta.name).await?; + raw_name.as_deref() + } else { + None + }, }, ); drop(reg); diff --git a/src/config.rs b/src/config.rs index 290e156..df1a0a2 100644 --- a/src/config.rs +++ b/src/config.rs @@ -93,6 +93,20 @@ pub struct DisplayDates { pub modification: bool, } +#[derive(Serialize, Deserialize, Default, Debug, Clone)] +#[serde(rename_all = "lowercase")] +pub enum Engine { + #[default] + Markdown, + Blag, +} + +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(default)] +pub struct BlagConfig { + pub bin: PathBuf, +} + #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct Config { @@ -100,12 +114,14 @@ pub struct Config { pub description: String, pub markdown_access: bool, pub js_enable: bool, + pub engine: Engine, pub style: StyleConfig, pub rss: RssConfig, pub dirs: DirsConfig, pub http: HttpConfig, pub render: RenderConfig, pub cache: CacheConfig, + pub blag: BlagConfig, } impl Default for Config { @@ -115,6 +131,7 @@ impl Default for Config { description: "blazingly fast markdown blog software written in rust memory safe".into(), markdown_access: true, js_enable: true, + engine: Default::default(), style: Default::default(), // i have a love-hate relationship with serde // it was engimatic at first, but then i started actually using it @@ -130,6 +147,7 @@ impl Default for Config { http: Default::default(), render: Default::default(), cache: Default::default(), + blag: Default::default(), } } } @@ -187,6 +205,12 @@ impl Default for CacheConfig { } } +impl Default for BlagConfig { + fn default() -> Self { + Self { bin: "blag".into() } + } +} + #[instrument(name = "config")] pub async fn load() -> Result { let config_file = env::var(format!( diff --git a/src/error.rs b/src/error.rs index 574f43e..dd49184 100644 --- a/src/error.rs +++ b/src/error.rs @@ -1,44 +1,42 @@ -use std::fmt::Display; - use askama_axum::Template; use axum::http::StatusCode; use axum::response::{IntoResponse, Response}; use thiserror::Error; use tracing::error; -#[derive(Debug)] -#[repr(transparent)] -pub struct FronmaError(fronma::error::Error); - -impl std::error::Error for FronmaError {} - -impl Display for FronmaError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("failed to parse front matter: ")?; - match &self.0 { - fronma::error::Error::MissingBeginningLine => f.write_str("missing beginning line"), - fronma::error::Error::MissingEndingLine => f.write_str("missing ending line"), - fronma::error::Error::SerdeYaml(yaml_error) => write!(f, "{}", yaml_error), - } - } -} - #[derive(Error, Debug)] #[allow(clippy::enum_variant_names)] pub enum PostError { #[error(transparent)] IoError(#[from] std::io::Error), - #[error(transparent)] - AskamaError(#[from] askama::Error), - #[error(transparent)] - ParseError(#[from] FronmaError), + #[error("{0}")] + ParseError(String), + #[error("{0}")] + RenderError(String), #[error("post {0:?} not found")] NotFound(String), } impl From for PostError { fn from(value: fronma::error::Error) -> Self { - Self::ParseError(FronmaError(value)) + let binding; + Self::ParseError(format!( + "failed to parse front matter: {}", + match value { + fronma::error::Error::MissingBeginningLine => "missing beginning line", + fronma::error::Error::MissingEndingLine => "missing ending line", + fronma::error::Error::SerdeYaml(yaml_error) => { + binding = yaml_error.to_string(); + &binding + } + } + )) + } +} + +impl From for PostError { + fn from(value: serde_json::Error) -> Self { + Self::ParseError(value.to_string()) } } diff --git a/src/main.rs b/src/main.rs index e531ed3..9dcee40 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,4 @@ -#![feature(let_chains, pattern)] +#![feature(let_chains, pattern, path_add_extension)] mod app; mod config; @@ -14,11 +14,13 @@ mod templates; use std::future::IntoFuture; use std::net::SocketAddr; +use std::path::PathBuf; use std::process::exit; use std::sync::Arc; use std::time::Duration; use color_eyre::eyre::{self, Context}; +use config::Engine; use tokio::net::TcpListener; use tokio::sync::RwLock; use tokio::task::JoinSet; @@ -32,7 +34,7 @@ use tracing_subscriber::{util::SubscriberInitExt, EnvFilter}; use crate::app::AppState; use crate::post::cache::{load_cache, CacheGuard, CACHE_VERSION}; -use crate::post::{MarkdownPosts, PostManager}; +use crate::post::{Blag, MarkdownPosts, PostManager}; use crate::templates::new_registry; use crate::templates::watcher::watch_templates; @@ -41,13 +43,7 @@ async fn main() -> eyre::Result<()> { color_eyre::install()?; let reg = tracing_subscriber::registry(); #[cfg(feature = "tokio-console")] - let reg = reg - .with( - EnvFilter::builder() - .with_default_directive(LevelFilter::TRACE.into()) - .from_env_lossy(), - ) - .with(console_subscriber::spawn()); + let reg = reg.with(console_subscriber::spawn()); #[cfg(not(feature = "tokio-console"))] let reg = reg.with( EnvFilter::builder() @@ -88,32 +84,39 @@ async fn main() -> eyre::Result<()> { .instrument(info_span!("custom_template_watcher")), ); - let cache = if config.cache.enable { - if config.cache.persistence && tokio::fs::try_exists(&config.cache.file).await? { - info!("loading cache from file"); - let mut cache = load_cache(&config.cache).await.unwrap_or_else(|err| { - error!("failed to load cache: {}", err); - info!("using empty cache"); - Default::default() - }); + let posts: Arc = match config.engine { + Engine::Markdown => { + let cache = if config.cache.enable { + if config.cache.persistence && tokio::fs::try_exists(&config.cache.file).await? { + info!("loading cache from file"); + let mut cache = load_cache(&config.cache).await.unwrap_or_else(|err| { + error!("failed to load cache: {}", err); + info!("using empty cache"); + Default::default() + }); - if cache.version() < CACHE_VERSION { - warn!("cache version changed, clearing cache"); - cache = Default::default(); - }; + if cache.version() < CACHE_VERSION { + warn!("cache version changed, clearing cache"); + cache = Default::default(); + }; - Some(cache) - } else { - Some(Default::default()) + Some(cache) + } else { + Some(Default::default()) + } + } else { + None + } + .map(|cache| CacheGuard::new(cache, config.cache.clone())) + .map(Arc::new); + + Arc::new(MarkdownPosts::new(Arc::clone(&config), cache.clone()).await?) } - } else { - None - } - .map(|cache| CacheGuard::new(cache, config.cache.clone())) - .map(Arc::new); - - let posts: Arc = - Arc::new(MarkdownPosts::new(Arc::clone(&config), cache.clone()).await?); + Engine::Blag => Arc::new(Blag::new( + config.dirs.posts.clone().into(), + Some(PathBuf::from("blag").into()), + )), + }; if config.cache.enable && config.cache.cleanup { if let Some(millis) = config.cache.cleanup_interval { diff --git a/src/post/blag.rs b/src/post/blag.rs new file mode 100644 index 0000000..b5632db --- /dev/null +++ b/src/post/blag.rs @@ -0,0 +1,182 @@ +use std::future::Future; +use std::mem; +use std::path::{Path, PathBuf}; +use std::pin::Pin; +use std::process::{ExitStatus, Stdio}; +use std::sync::Arc; + +use axum::async_trait; +use axum::http::HeaderValue; +use futures::stream::FuturesUnordered; +use futures::StreamExt; +use tokio::fs::OpenOptions; +use tokio::io::{AsyncBufReadExt, AsyncReadExt, BufReader}; +use tokio::time::Instant; +use tracing::{debug, error}; + +use crate::error::PostError; +use crate::post::Filter; + +use super::{ApplyFilters, PostManager, PostMetadata, RenderStats, ReturnedPost}; + +pub struct Blag { + root: Arc, + blag_bin: Arc, +} + +impl Blag { + pub fn new(root: Arc, blag_bin: Option>) -> Blag { + Self { + root, + blag_bin: blag_bin.unwrap_or_else(|| PathBuf::from("blag").into()), + } + } +} + +#[async_trait] +impl PostManager for Blag { + async fn get_all_posts( + &self, + filters: &[Filter<'_>], + ) -> Result, PostError> { + let mut set = FuturesUnordered::new(); + let mut meow = Vec::new(); + let mut files = tokio::fs::read_dir(&self.root).await?; + + while let Ok(Some(entry)) = files.next_entry().await { + let file_type = entry.file_type().await?; + if file_type.is_file() { + let name = entry.file_name().into_string().unwrap(); + + if name.ends_with(".sh") { + set.push(async move { self.get_post(name.trim_end_matches(".sh")).await }); + } + } + } + + while let Some(result) = set.next().await { + let post = match result { + Ok(v) => match v { + ReturnedPost::Rendered(meta, content, stats) => (meta, content, stats), + ReturnedPost::Raw(..) => unreachable!(), + }, + Err(err) => { + error!("error while rendering blagpost: {err}"); + continue; + } + }; + + if post.0.apply_filters(filters) { + meow.push(post); + } + } + + debug!("collected posts"); + + Ok(meow) + } + + async fn get_post(&self, name: &str) -> Result { + let mut path = self.root.join(name); + + if name.ends_with(".sh") { + let mut buf = Vec::new(); + let mut file = + OpenOptions::new() + .read(true) + .open(&path) + .await + .map_err(|err| match err.kind() { + std::io::ErrorKind::NotFound => PostError::NotFound(name.to_string()), + _ => PostError::IoError(err), + })?; + file.read_to_end(&mut buf).await?; + + return Ok(ReturnedPost::Raw( + buf, + HeaderValue::from_static("text/x-shellscript"), + )); + } else { + path.add_extension("sh"); + } + + let start = Instant::now(); + let stat = tokio::fs::metadata(&path) + .await + .map_err(|err| match err.kind() { + std::io::ErrorKind::NotFound => PostError::NotFound(name.to_string()), + _ => PostError::IoError(err), + })?; + + if !stat.is_file() { + return Err(PostError::NotFound(name.to_string())); + } + + let mut cmd = tokio::process::Command::new(&*self.blag_bin) + .arg(path) + .stdout(Stdio::piped()) + .spawn()?; + + let stdout = cmd.stdout.take().unwrap(); + + let mut reader = BufReader::new(stdout); + let mut buf = String::new(); + reader.read_line(&mut buf).await?; + + let mut meta: PostMetadata = serde_json::from_str(&buf)?; + meta.name = name.to_string(); + + enum Return { + Read(String), + Exit(ExitStatus), + } + + let mut futures: FuturesUnordered< + Pin> + Send>>, + > = FuturesUnordered::new(); + + buf.clear(); + let mut fut_buf = mem::take(&mut buf); + + futures.push(Box::pin(async move { + reader + .read_to_string(&mut fut_buf) + .await + .map(|_| Return::Read(fut_buf)) + })); + futures.push(Box::pin(async move { cmd.wait().await.map(Return::Exit) })); + + while let Some(res) = futures.next().await { + match res? { + Return::Read(fut_buf) => { + buf = fut_buf; + debug!("read output: {} bytes", buf.len()); + } + Return::Exit(exit_status) => { + debug!("exited: {exit_status}"); + if !exit_status.success() { + return Err(PostError::RenderError(exit_status.to_string())); + } + } + } + } + + drop(futures); + + let elapsed = start.elapsed(); + + Ok(ReturnedPost::Rendered( + meta, + buf, + RenderStats::ParsedAndRendered(elapsed, elapsed, elapsed), + )) + } + + async fn get_raw(&self, name: &str) -> Result, PostError> { + let mut buf = String::with_capacity(name.len() + 3); + buf += name; + buf += ".sh"; + + Ok(Some(buf)) + } +} diff --git a/src/post/markdown_posts.rs b/src/post/markdown_posts.rs index 50547e6..71414e6 100644 --- a/src/post/markdown_posts.rs +++ b/src/post/markdown_posts.rs @@ -286,4 +286,12 @@ impl PostManager for MarkdownPosts { .await } } + + async fn get_raw(&self, name: &str) -> Result, PostError> { + let mut buf = String::with_capacity(name.len() + 3); + buf += name; + buf += ".md"; + + Ok(Some(buf)) + } } diff --git a/src/post/mod.rs b/src/post/mod.rs index c501746..9009ca4 100644 --- a/src/post/mod.rs +++ b/src/post/mod.rs @@ -1,3 +1,4 @@ +pub mod blag; pub mod cache; pub mod markdown_posts; @@ -8,8 +9,10 @@ use chrono::{DateTime, Utc}; use serde::{Deserialize, Serialize}; use crate::error::PostError; -pub use crate::post::markdown_posts::MarkdownPosts; +pub use blag::Blag; +pub use markdown_posts::MarkdownPosts; +// TODO: replace String with Arc #[derive(Serialize, Deserialize, Clone, Debug)] pub struct PostMetadata { pub name: String, @@ -24,7 +27,7 @@ pub struct PostMetadata { pub tags: Vec, } -#[derive(Serialize)] +#[derive(Serialize, Debug)] pub enum RenderStats { Cached(Duration), // format: Total, Parsed in, Rendered in @@ -41,7 +44,7 @@ pub enum Filter<'a> { Tags(&'a [&'a str]), } -impl<'a> Filter<'a> { +impl Filter<'_> { pub fn apply(&self, meta: &PostMetadata) -> bool { match self { Filter::Tags(tags) => tags @@ -110,5 +113,10 @@ pub trait PostManager { async fn get_post(&self, name: &str) -> Result; - async fn cleanup(&self); + async fn cleanup(&self) {} + + #[allow(unused)] + async fn get_raw(&self, name: &str) -> Result, PostError> { + Ok(None) + } } diff --git a/templates/footer.hbs b/templates/footer.hbs index d50a79c..30ef8ad 100644 --- a/templates/footer.hbs +++ b/templates/footer.hbs @@ -15,7 +15,7 @@ running {{bingus_info.name} {{duration this}} {{/if}} {{/each}} -{{#if markdown_access}} +{{#if raw_name}} - - view raw + view raw {{/if}}