diff --git a/Cargo.lock b/Cargo.lock index fd89cfb..faf9699 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -302,6 +302,7 @@ dependencies = [ "color-eyre", "comrak", "console-subscriber", + "derive_more", "fronma", "rss", "scc", @@ -486,6 +487,12 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "crc32fast" version = "1.4.0" @@ -585,6 +592,19 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version", + "syn 1.0.109", +] + [[package]] name = "deunicode" version = "1.4.4" @@ -1435,6 +1455,15 @@ version = "0.1.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustversion" version = "1.0.15" @@ -1472,6 +1501,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b84345e4c9bd703274a082fb80caaa99b7612be48dfaa1dd9266577ec412309d" +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + [[package]] name = "serde" version = "1.0.198" diff --git a/Cargo.toml b/Cargo.toml index 661cd3c..b708cc5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -36,6 +36,7 @@ comrak = { version = "0.22.0", features = [ "syntect", ], default-features = false } console-subscriber = { version = "0.2.0", optional = true } +derive_more = "0.99.17" fronma = "0.2.0" rss = "2.0.7" scc = { version = "2.1.0", features = ["serde"] } diff --git a/README.md b/README.md index de24d7f..dd87090 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ blazingly fast markdown blog software written in rust memory safe - [ ] ^ replace HashMap with HashCache once i implement [this](https://github.com/wvwwvwwv/scalable-concurrent-containers/issues/139) - [x] (de)compress cache with zstd on startup/shutdown - [ ] make date parsing less strict -- [ ] make date formatting better +- [x] make date formatting better - [ ] date formatting respects user timezone - [x] clean up imports and require less features - [ ] improve home page @@ -36,9 +36,16 @@ blazingly fast markdown blog software written in rust memory safe the default configuration with comments looks like this ```toml -title = "bingus-blog" # title of the website -description = "blazingly fast markdown blog software written in rust memory safe" # description of the website -raw_access = true # allow users to see the raw markdown of a post +title = "bingus-blog" # title of the blog +# description of the blog +description = "blazingly fast markdown blog software written in rust memory safe" +markdown_access = true # allow users to see the raw markdown of a post + # endpoint: /posts/.md +date_format = "RFC3339" # format string used to format dates in the backend + # it's highly recommended to leave this as default, + # so the date can be formatted by the browser. + # format: https://docs.rs/chrono/latest/chrono/format/strftime/index.html#specifiers +js_enable = true # enable javascript (required for above) [rss] enable = false # serve an rss field under /feed.xml diff --git a/src/app.rs b/src/app.rs new file mode 100644 index 0000000..1a8b0ca --- /dev/null +++ b/src/app.rs @@ -0,0 +1,198 @@ +use std::sync::Arc; +use std::time::Duration; + +use askama_axum::Template; +use axum::extract::{Path, Query, State}; +use axum::http::header::CONTENT_TYPE; +use axum::http::{header, Request}; +use axum::response::{IntoResponse, Redirect, Response}; +use axum::routing::get; +use axum::{Json, Router}; +use rss::{Category, ChannelBuilder, ItemBuilder}; +use serde::Deserialize; +use tower_http::services::ServeDir; +use tower_http::trace::TraceLayer; +use tracing::{info, info_span, Span}; + +use crate::config::{Config, DateFormat}; +use crate::error::{AppError, AppResult}; +use crate::filters; +use crate::post::{MarkdownPosts, PostManager, PostMetadata, RenderStats, ReturnedPost}; + +#[derive(Clone)] +pub struct AppState { + pub config: Arc, + pub posts: Arc>>, +} + +#[derive(Template)] +#[template(path = "index.html")] +struct IndexTemplate { + title: String, + description: String, + posts: Vec, + df: DateFormat, + js: bool, +} + +#[derive(Template)] +#[template(path = "post.html")] +struct PostTemplate { + meta: PostMetadata, + rendered: String, + rendered_in: RenderStats, + markdown_access: bool, + df: DateFormat, + js: bool, +} + +#[derive(Deserialize)] +struct QueryParams { + tag: Option, + #[serde(rename = "n")] + num_posts: Option, +} + +async fn index( + State(AppState { config, posts }): State, + Query(query): Query, +) -> AppResult { + let posts = posts + .get_max_n_post_metadata_with_optional_tag_sorted(query.num_posts, query.tag.as_ref()) + .await?; + + Ok(IndexTemplate { + title: config.title.clone(), + description: config.description.clone(), + posts, + df: config.date_format.clone(), + js: config.js_enable, + }) +} + +async fn all_posts( + State(AppState { posts, .. }): State, + Query(query): Query, +) -> AppResult>> { + let posts = posts + .get_max_n_post_metadata_with_optional_tag_sorted(query.num_posts, query.tag.as_ref()) + .await?; + + Ok(Json(posts)) +} + +async fn rss( + State(AppState { config, posts }): State, + Query(query): Query, +) -> AppResult { + if !config.rss.enable { + return Err(AppError::RssDisabled); + } + + let posts = posts + .get_all_posts(|metadata, _| { + !query + .tag + .as_ref() + .is_some_and(|tag| !metadata.tags.contains(tag)) + }) + .await?; + + let mut channel = ChannelBuilder::default(); + channel + .title(&config.title) + .link(config.rss.link.to_string()) + .description(&config.description); + //TODO: .language() + + for (metadata, content, _) in posts { + channel.item( + ItemBuilder::default() + .title(metadata.title) + .description(metadata.description) + .author(metadata.author) + .categories( + metadata + .tags + .into_iter() + .map(|tag| Category { + name: tag, + domain: None, + }) + .collect::>(), + ) + .pub_date(metadata.created_at.map(|date| date.to_rfc2822())) + .content(content) + .link( + config + .rss + .link + .join(&format!("/posts/{}", metadata.name))? + .to_string(), + ) + .build(), + ); + } + + let body = channel.build().to_string(); + drop(channel); + + Ok(([(header::CONTENT_TYPE, "text/xml")], body).into_response()) +} + +async fn post( + State(AppState { config, posts }): State, + Path(name): Path, +) -> AppResult { + match posts.get_post(&name).await? { + ReturnedPost::Rendered(meta, rendered, rendered_in) => { + let page = PostTemplate { + meta, + rendered, + rendered_in, + markdown_access: config.markdown_access, + df: config.date_format.clone(), + js: config.js_enable, + }; + + Ok(page.into_response()) + } + ReturnedPost::Raw(body, content_type) => { + Ok(([(CONTENT_TYPE, content_type)], body).into_response()) + } + } +} + +pub fn new(config: &Config) -> Router { + Router::new() + .route("/", get(index)) + .route( + "/post/:name", + get( + |Path(name): Path| async move { Redirect::to(&format!("/posts/{}", name)) }, + ), + ) + .route("/posts/:name", get(post)) + .route("/posts", get(all_posts)) + .route("/feed.xml", get(rss)) + .nest_service( + "/static", + ServeDir::new(&config.dirs._static).precompressed_gzip(), + ) + .nest_service("/media", ServeDir::new(&config.dirs.media)) + .layer( + TraceLayer::new_for_http() + .make_span_with(|request: &Request<_>| { + info_span!( + "request", + method = ?request.method(), + path = ?request.uri().path(), + ) + }) + .on_response(|response: &Response<_>, duration: Duration, span: &Span| { + let _ = span.enter(); + let status = response.status(); + info!(?status, ?duration, "response"); + }), + ) +} diff --git a/src/config.rs b/src/config.rs index f22e4fd..0906af9 100644 --- a/src/config.rs +++ b/src/config.rs @@ -1,11 +1,11 @@ use std::env; -use std::net::{IpAddr, Ipv4Addr}; +use std::net::{IpAddr, Ipv6Addr}; use std::path::PathBuf; use color_eyre::eyre::{bail, Context, Result}; use serde::{Deserialize, Serialize}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; -use tracing::{error, info}; +use tracing::{error, info, instrument}; use url::Url; use crate::ranged_i128_visitor::RangedI128Visitor; @@ -49,6 +49,8 @@ pub struct HttpConfig { pub struct DirsConfig { pub posts: PathBuf, pub media: PathBuf, + #[serde(rename = "static")] + pub _static: PathBuf, } #[derive(Serialize, Deserialize, Debug, Clone)] @@ -57,13 +59,22 @@ pub struct RssConfig { pub link: Url, } +#[derive(Serialize, Deserialize, Debug, Clone, Default)] +pub enum DateFormat { + #[default] + RFC3339, + #[serde(untagged)] + Strftime(String), +} + #[derive(Serialize, Deserialize, Debug, Clone)] #[serde(default)] pub struct Config { pub title: String, pub description: String, - pub raw_access: bool, - pub num_posts: usize, + pub markdown_access: bool, + pub date_format: DateFormat, + pub js_enable: bool, pub rss: RssConfig, pub dirs: DirsConfig, pub http: HttpConfig, @@ -76,8 +87,9 @@ impl Default for Config { Self { title: "bingus-blog".into(), description: "blazingly fast markdown blog software written in rust memory safe".into(), - raw_access: true, - num_posts: 5, + markdown_access: true, + date_format: Default::default(), + js_enable: true, // i have a love-hate relationship with serde // it was engimatic at first, but then i started actually using it // writing my own serialize and deserialize implementations.. spending @@ -101,6 +113,7 @@ impl Default for DirsConfig { Self { posts: "posts".into(), media: "media".into(), + _static: "static".into(), } } } @@ -108,7 +121,7 @@ impl Default for DirsConfig { impl Default for HttpConfig { fn default() -> Self { Self { - host: IpAddr::V4(Ipv4Addr::UNSPECIFIED), + host: IpAddr::V6(Ipv6Addr::UNSPECIFIED), port: 3000, } } @@ -138,6 +151,7 @@ impl Default for CacheConfig { } } +#[instrument(name = "config")] pub async fn load() -> Result { let config_file = env::var(format!( "{}_CONFIG", diff --git a/src/filters.rs b/src/filters.rs index 35929dc..24d0a10 100644 --- a/src/filters.rs +++ b/src/filters.rs @@ -1,11 +1,29 @@ -use std::{collections::HashMap, time::Duration}; +use std::collections::HashMap; +use std::fmt::Display; +use std::time::Duration; use chrono::{DateTime, TimeZone}; +use crate::config::DateFormat; use crate::post::PostMetadata; -pub fn date(date: &DateTime) -> Result { - Ok(date.to_rfc3339_opts(chrono::SecondsFormat::Secs, true)) +fn format_date(date: &DateTime, date_format: &DateFormat) -> String +where + T: TimeZone, + T::Offset: Display, +{ + match date_format { + DateFormat::RFC3339 => date.to_rfc3339_opts(chrono::SecondsFormat::Secs, true), + DateFormat::Strftime(ref format_string) => date.format(format_string).to_string(), + } +} + +pub fn date(date: &DateTime, date_format: &DateFormat) -> Result +where + T: TimeZone, + T::Offset: Display, +{ + Ok(format_date(date, date_format)) } pub fn duration(duration: &&Duration) -> Result { diff --git a/src/main.rs b/src/main.rs index ff09d4c..c352c95 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ #![feature(let_chains)] +mod app; mod config; mod error; mod filters; @@ -10,184 +11,23 @@ mod ranged_i128_visitor; mod systemtime_as_secs; use std::future::IntoFuture; -use std::io::Read; use std::net::SocketAddr; use std::process::exit; use std::sync::Arc; use std::time::Duration; -use askama_axum::Template; -use axum::extract::{Path, Query, State}; -use axum::http::{header, Request}; -use axum::response::{IntoResponse, Redirect, Response}; -use axum::routing::{get, Router}; -use axum::Json; use color_eyre::eyre::{self, Context}; -use error::AppError; -use rss::{Category, ChannelBuilder, ItemBuilder}; -use serde::Deserialize; -use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::net::TcpListener; use tokio::task::JoinSet; use tokio::{select, signal}; use tokio_util::sync::CancellationToken; -use tower_http::services::ServeDir; -use tower_http::trace::TraceLayer; use tracing::level_filters::LevelFilter; -use tracing::{debug, error, info, info_span, warn, Span}; -use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilter}; +use tracing::{debug, info, warn}; +use tracing_subscriber::layer::SubscriberExt; +use tracing_subscriber::{util::SubscriberInitExt, EnvFilter}; -use crate::config::Config; -use crate::error::{AppResult, PostError}; -use crate::post::cache::{Cache, CACHE_VERSION}; -use crate::post::{PostManager, PostMetadata, RenderStats}; - -type ArcState = Arc; - -#[derive(Clone)] -struct AppState { - pub config: Config, - pub posts: PostManager, -} - -#[derive(Template)] -#[template(path = "index.html")] -struct IndexTemplate { - title: String, - description: String, - posts: Vec, - rss: bool, -} - -#[derive(Template)] -#[template(path = "post.html")] -struct PostTemplate { - meta: PostMetadata, - rendered: String, - rendered_in: RenderStats, - markdown_access: bool, -} - -#[derive(Deserialize)] -struct QueryParams { - tag: Option, - #[serde(rename = "n")] - num_posts: Option, -} - -async fn index( - State(state): State, - Query(query): Query, -) -> AppResult { - let posts = state - .posts - .get_max_n_post_metadata_with_optional_tag_sorted(query.num_posts, query.tag.as_ref()) - .await?; - - Ok(IndexTemplate { - title: state.config.title.clone(), - description: state.config.description.clone(), - posts, - rss: state.config.rss.enable, - }) -} - -async fn all_posts( - State(state): State, - Query(query): Query, -) -> AppResult>> { - let posts = state - .posts - .get_max_n_post_metadata_with_optional_tag_sorted(query.num_posts, query.tag.as_ref()) - .await?; - - Ok(Json(posts)) -} - -async fn rss( - State(state): State, - Query(query): Query, -) -> AppResult { - if !state.config.rss.enable { - return Err(AppError::RssDisabled); - } - - let posts = state - .posts - .get_all_posts_filtered(|metadata, _| { - !query - .tag - .as_ref() - .is_some_and(|tag| !metadata.tags.contains(tag)) - }) - .await?; - - let mut channel = ChannelBuilder::default(); - channel - .title(&state.config.title) - .link(state.config.rss.link.to_string()) - .description(&state.config.description); - //TODO: .language() - - for (metadata, content, _) in posts { - channel.item( - ItemBuilder::default() - .title(metadata.title) - .description(metadata.description) - .author(metadata.author) - .categories( - metadata - .tags - .into_iter() - .map(|tag| Category { - name: tag, - domain: None, - }) - .collect::>(), - ) - .pub_date(metadata.created_at.map(|date| date.to_rfc2822())) - .content(content) - .link( - state - .config - .rss - .link - .join(&format!("/posts/{}", metadata.name))? - .to_string(), - ) - .build(), - ); - } - - let body = channel.build().to_string(); - drop(channel); - - Ok(([(header::CONTENT_TYPE, "text/xml")], body).into_response()) -} - -async fn post(State(state): State, Path(name): Path) -> AppResult { - if name.ends_with(".md") && state.config.raw_access { - let mut file = tokio::fs::OpenOptions::new() - .read(true) - .open(state.config.dirs.posts.join(&name)) - .await?; - - let mut buf = Vec::new(); - file.read_to_end(&mut buf).await?; - - Ok(([("content-type", "text/plain")], buf).into_response()) - } else { - let post = state.posts.get_post(&name).await?; - let page = PostTemplate { - meta: post.0, - rendered: post.1, - rendered_in: post.2, - markdown_access: state.config.raw_access, - }; - - Ok(page.into_response()) - } -} +use crate::app::AppState; +use crate::post::{MarkdownPosts, PostManager}; #[tokio::main] async fn main() -> eyre::Result<()> { @@ -204,89 +44,26 @@ async fn main() -> eyre::Result<()> { .with(tracing_subscriber::fmt::layer()) .init(); - let config = config::load() - .await - .context("couldn't load configuration")?; + let config = Arc::new( + config::load() + .await + .context("couldn't load configuration")?, + ); let socket_addr = SocketAddr::new(config.http.host, config.http.port); let mut tasks = JoinSet::new(); let cancellation_token = CancellationToken::new(); - let posts = if config.cache.enable { - if config.cache.persistence - && tokio::fs::try_exists(&config.cache.file) - .await - .with_context(|| { - format!("failed to check if {} exists", config.cache.file.display()) - })? - { - info!("loading cache from file"); - let path = &config.cache.file; - let load_cache = async { - let mut cache_file = tokio::fs::File::open(&path) - .await - .context("failed to open cache file")?; - let serialized = if config.cache.compress { - let cache_file = cache_file.into_std().await; - tokio::task::spawn_blocking(move || { - let mut buf = Vec::with_capacity(4096); - zstd::stream::read::Decoder::new(cache_file)?.read_to_end(&mut buf)?; - Ok::<_, std::io::Error>(buf) - }) - .await - .context("failed to join blocking thread")? - .context("failed to read cache file")? - } else { - let mut buf = Vec::with_capacity(4096); - cache_file - .read_to_end(&mut buf) - .await - .context("failed to read cache file")?; - buf - }; - let mut cache: Cache = - bitcode::deserialize(serialized.as_slice()).context("failed to parse cache")?; - if cache.version() < CACHE_VERSION { - warn!("cache version changed, clearing cache"); - cache = Cache::default(); - }; - - Ok::(PostManager::new_with_cache( - config.dirs.posts.clone(), - config.render.clone(), - cache, - )) - } - .await; - match load_cache { - Ok(posts) => posts, - Err(err) => { - error!("failed to load cache: {}", err); - info!("using empty cache"); - PostManager::new_with_cache( - config.dirs.posts.clone(), - config.render.clone(), - Default::default(), - ) - } - } - } else { - PostManager::new_with_cache( - config.dirs.posts.clone(), - config.render.clone(), - Default::default(), - ) - } - } else { - PostManager::new(config.dirs.posts.clone(), config.render.clone()) + let posts = Arc::new(MarkdownPosts::new(Arc::clone(&config)).await?); + let state = AppState { + config: Arc::clone(&config), + posts: Arc::clone(&posts), }; - let state = Arc::new(AppState { config, posts }); - - if state.config.cache.enable && state.config.cache.cleanup { - if let Some(t) = state.config.cache.cleanup_interval { - let state = Arc::clone(&state); + if config.cache.enable && config.cache.cleanup { + if let Some(t) = config.cache.cleanup_interval { + let posts = Arc::clone(&posts); let token = cancellation_token.child_token(); debug!("setting up cleanup task"); tasks.spawn(async move { @@ -295,45 +72,17 @@ async fn main() -> eyre::Result<()> { select! { _ = token.cancelled() => break, _ = interval.tick() => { - state.posts.cleanup().await + posts.cleanup().await } } } }); } else { - state.posts.cleanup().await; + posts.cleanup().await; } } - let app = Router::new() - .route("/", get(index)) - .route( - "/post/:name", - get( - |Path(name): Path| async move { Redirect::to(&format!("/posts/{}", name)) }, - ), - ) - .route("/posts/:name", get(post)) - .route("/posts", get(all_posts)) - .route("/feed.xml", get(rss)) - .nest_service("/static", ServeDir::new("static").precompressed_gzip()) - .nest_service("/media", ServeDir::new("media")) - .layer( - TraceLayer::new_for_http() - .make_span_with(|request: &Request<_>| { - info_span!( - "request", - method = ?request.method(), - path = ?request.uri().path(), - ) - }) - .on_response(|response: &Response<_>, duration: Duration, span: &Span| { - let _ = span.enter(); - let status = response.status(); - info!(?status, ?duration, "response"); - }), - ) - .with_state(state.clone()); + let app = app::new(&config).with_state(state.clone()); let listener = TcpListener::bind(socket_addr) .await @@ -381,36 +130,7 @@ async fn main() -> eyre::Result<()> { task.context("failed to join task")?; } - // write cache to file - let config = &state.config; - let posts = &state.posts; - if config.cache.enable - && config.cache.persistence - && let Some(cache) = posts.cache() - { - let path = &config.cache.file; - let serialized = bitcode::serialize(cache).context("failed to serialize cache")?; - let mut cache_file = tokio::fs::File::create(path) - .await - .with_context(|| format!("failed to open cache at {}", path.display()))?; - let compression_level = config.cache.compression_level; - if config.cache.compress { - let cache_file = cache_file.into_std().await; - tokio::task::spawn_blocking(move || { - std::io::Write::write_all( - &mut zstd::stream::write::Encoder::new(cache_file, compression_level)? - .auto_finish(), - &serialized, - ) - }) - .await - .context("failed to join blocking thread")? - } else { - cache_file.write_all(&serialized).await - } - .context("failed to write cache to file")?; - info!("wrote cache to {}", path.display()); - } + drop(state); Ok::<(), color_eyre::Report>(()) }; diff --git a/src/post/cache.rs b/src/post/cache.rs index f6b86cc..cd3e837 100644 --- a/src/post/cache.rs +++ b/src/post/cache.rs @@ -1,12 +1,14 @@ use std::hash::{DefaultHasher, Hash, Hasher}; +use std::io::Read; +use crate::config::{Config, RenderConfig}; +use crate::post::PostMetadata; +use color_eyre::eyre::{self, Context}; use scc::HashMap; use serde::{Deserialize, Serialize}; +use tokio::io::AsyncReadExt; use tracing::{debug, instrument}; -use crate::config::RenderConfig; -use crate::post::PostMetadata; - /// do not persist cache if this version number changed pub const CACHE_VERSION: u16 = 2; @@ -133,3 +135,29 @@ impl Cache { self.1 } } + +pub(crate) async fn load_cache(config: &Config) -> Result { + let path = &config.cache.file; + let mut cache_file = tokio::fs::File::open(&path) + .await + .context("failed to open cache file")?; + let serialized = if config.cache.compress { + let cache_file = cache_file.into_std().await; + tokio::task::spawn_blocking(move || { + let mut buf = Vec::with_capacity(4096); + zstd::stream::read::Decoder::new(cache_file)?.read_to_end(&mut buf)?; + Ok::<_, std::io::Error>(buf) + }) + .await? + .context("failed to read cache file")? + } else { + let mut buf = Vec::with_capacity(4096); + cache_file + .read_to_end(&mut buf) + .await + .context("failed to read cache file")?; + buf + }; + + bitcode::deserialize(serialized.as_slice()).context("failed to parse cache") +} diff --git a/src/post/markdown_posts.rs b/src/post/markdown_posts.rs new file mode 100644 index 0000000..e1ee1df --- /dev/null +++ b/src/post/markdown_posts.rs @@ -0,0 +1,344 @@ +use std::collections::BTreeSet; +use std::io::{self, Write}; +use std::ops::Deref; +use std::path::Path; +use std::time::Duration; +use std::time::Instant; +use std::time::SystemTime; + +use axum::http::HeaderValue; +use chrono::{DateTime, Utc}; +use color_eyre::eyre::{self, Context}; +use fronma::parser::{parse, ParsedData}; +use serde::Deserialize; +use tokio::fs; +use tokio::io::AsyncReadExt; +use tracing::{error, info, warn}; + +use crate::config::Config; +use crate::markdown_render::render; +use crate::post::cache::{load_cache, Cache, CACHE_VERSION}; +use crate::post::{PostError, PostManager, PostMetadata, RenderStats, ReturnedPost}; +use crate::systemtime_as_secs::as_secs; + +#[derive(Deserialize)] +struct FrontMatter { + pub title: String, + pub description: String, + pub author: String, + pub icon: Option, + pub created_at: Option>, + pub modified_at: Option>, + #[serde(default)] + pub tags: BTreeSet, +} + +impl FrontMatter { + pub fn into_full( + self, + name: String, + created: Option, + modified: Option, + ) -> PostMetadata { + PostMetadata { + name, + title: self.title, + description: self.description, + author: self.author, + icon: self.icon, + created_at: self.created_at.or_else(|| created.map(|t| t.into())), + modified_at: self.modified_at.or_else(|| modified.map(|t| t.into())), + tags: self.tags.into_iter().collect(), + } + } +} +pub struct MarkdownPosts +where + C: Deref, +{ + cache: Option, + config: C, +} + +impl MarkdownPosts +where + C: Deref, +{ + pub async fn new(config: C) -> eyre::Result> { + if config.cache.enable { + if config.cache.persistence && tokio::fs::try_exists(&config.cache.file).await? { + info!("loading cache from file"); + let mut cache = load_cache(&config).await.unwrap_or_else(|err| { + error!("failed to load cache: {}", err); + info!("using empty cache"); + Default::default() + }); + + if cache.version() < CACHE_VERSION { + warn!("cache version changed, clearing cache"); + cache = Default::default(); + }; + + Ok(Self { + cache: Some(cache), + config, + }) + } else { + Ok(Self { + cache: Some(Default::default()), + config, + }) + } + } else { + Ok(Self { + cache: None, + config, + }) + } + } + + async fn parse_and_render( + &self, + name: String, + path: impl AsRef, + ) -> Result<(PostMetadata, String, (Duration, Duration)), PostError> { + let parsing_start = Instant::now(); + let mut file = match tokio::fs::OpenOptions::new().read(true).open(&path).await { + Ok(val) => val, + Err(err) => match err.kind() { + io::ErrorKind::NotFound => return Err(PostError::NotFound(name)), + _ => return Err(PostError::IoError(err)), + }, + }; + let stat = file.metadata().await?; + let modified = stat.modified()?; + let created = stat.created().ok(); + + let mut content = String::with_capacity(stat.len() as usize); + file.read_to_string(&mut content).await?; + + let ParsedData { headers, body } = parse::(&content)?; + let metadata = headers.into_full(name.to_owned(), created, Some(modified)); + let parsing = parsing_start.elapsed(); + + let before_render = Instant::now(); + let post = render(body, &self.config.render); + let rendering = before_render.elapsed(); + + if let Some(cache) = self.cache.as_ref() { + cache + .insert( + name.to_string(), + metadata.clone(), + as_secs(&modified), + post.clone(), + &self.config.render, + ) + .await + .unwrap_or_else(|err| warn!("failed to insert {:?} into cache", err.0)) + }; + + Ok((metadata, post, (parsing, rendering))) + } + + fn cache(&self) -> Option<&Cache> { + self.cache.as_ref() + } + + fn try_drop(&mut self) -> Result<(), eyre::Report> { + // write cache to file + let config = &self.config.cache; + if config.enable + && config.persistence + && let Some(cache) = self.cache() + { + let path = &config.file; + let serialized = bitcode::serialize(cache).context("failed to serialize cache")?; + let mut cache_file = std::fs::File::create(path) + .with_context(|| format!("failed to open cache at {}", path.display()))?; + let compression_level = config.compression_level; + if config.compress { + std::io::Write::write_all( + &mut zstd::stream::write::Encoder::new(cache_file, compression_level)? + .auto_finish(), + &serialized, + ) + } else { + cache_file.write_all(&serialized) + } + .context("failed to write cache to file")?; + info!("wrote cache to {}", path.display()); + } + Ok(()) + } +} + +impl Drop for MarkdownPosts +where + C: Deref, +{ + fn drop(&mut self) { + self.try_drop().unwrap() + } +} + +impl PostManager for MarkdownPosts +where + C: Deref, +{ + async fn get_all_post_metadata( + &self, + filter: impl Fn(&PostMetadata) -> bool, + ) -> Result, PostError> { + let mut posts = Vec::new(); + + let mut read_dir = fs::read_dir(&self.config.dirs.posts).await?; + while let Some(entry) = read_dir.next_entry().await? { + let path = entry.path(); + let stat = fs::metadata(&path).await?; + + if stat.is_file() && path.extension().is_some_and(|ext| ext == "md") { + let mtime = as_secs(&stat.modified()?); + // TODO. this? + let name = path + .clone() + .file_stem() + .unwrap() + .to_string_lossy() + .to_string(); + + if let Some(cache) = self.cache.as_ref() + && let Some(hit) = cache.lookup_metadata(&name, mtime).await + && filter(&hit) + { + posts.push(hit); + } else { + match self.parse_and_render(name, path).await { + Ok((metadata, ..)) => { + if filter(&metadata) { + posts.push(metadata); + } + } + Err(err) => match err { + PostError::IoError(ref io_err) + if matches!(io_err.kind(), io::ErrorKind::NotFound) => + { + warn!("TOCTOU: {}", err) + } + _ => return Err(err), + }, + } + } + } + } + + Ok(posts) + } + + async fn get_all_posts( + &self, + filter: impl Fn(&PostMetadata, &str) -> bool, + ) -> Result, PostError> { + let mut posts = Vec::new(); + + let mut read_dir = fs::read_dir(&self.config.dirs.posts).await?; + while let Some(entry) = read_dir.next_entry().await? { + let path = entry.path(); + let stat = fs::metadata(&path).await?; + + if stat.is_file() && path.extension().is_some_and(|ext| ext == "md") { + let name = path + .clone() + .file_stem() + .unwrap() + .to_string_lossy() + .to_string(); + + let post = self.get_post(&name).await?; + if let ReturnedPost::Rendered(meta, content, stats) = post + && filter(&meta, &content) + { + posts.push((meta, content, stats)); + } + } + } + + Ok(posts) + } + + async fn get_post(&self, name: &str) -> Result { + if self.config.markdown_access && name.ends_with(".md") { + let path = self.config.dirs.posts.join(name); + + let mut file = match tokio::fs::OpenOptions::new().read(true).open(&path).await { + Ok(value) => value, + Err(err) => match err.kind() { + io::ErrorKind::NotFound => { + if let Some(cache) = self.cache.as_ref() { + cache.remove(name).await; + } + return Err(PostError::NotFound(name.to_string())); + } + _ => return Err(PostError::IoError(err)), + }, + }; + + let mut buf = Vec::with_capacity(4096); + + file.read_to_end(&mut buf).await?; + + Ok(ReturnedPost::Raw( + buf, + HeaderValue::from_static("text/plain"), + )) + } else { + let start = Instant::now(); + let path = self.config.dirs.posts.join(name.to_owned() + ".md"); + + let stat = match tokio::fs::metadata(&path).await { + Ok(value) => value, + Err(err) => match err.kind() { + io::ErrorKind::NotFound => { + if let Some(cache) = self.cache.as_ref() { + cache.remove(name).await; + } + return Err(PostError::NotFound(name.to_string())); + } + _ => return Err(PostError::IoError(err)), + }, + }; + let mtime = as_secs(&stat.modified()?); + + if let Some(cache) = self.cache.as_ref() + && let Some(hit) = cache.lookup(name, mtime, &self.config.render).await + { + Ok(ReturnedPost::Rendered( + hit.metadata, + hit.rendered, + RenderStats::Cached(start.elapsed()), + )) + } else { + let (metadata, rendered, stats) = + self.parse_and_render(name.to_string(), path).await?; + Ok(ReturnedPost::Rendered( + metadata, + rendered, + RenderStats::ParsedAndRendered(start.elapsed(), stats.0, stats.1), + )) + } + } + } + + async fn cleanup(&self) { + if let Some(cache) = self.cache.as_ref() { + cache + .cleanup(|name| { + std::fs::metadata(self.config.dirs.posts.join(name.to_owned() + ".md")) + .ok() + .and_then(|metadata| metadata.modified().ok()) + .map(|mtime| as_secs(&mtime)) + }) + .await + } + } +} diff --git a/src/post/mod.rs b/src/post/mod.rs index 1e2619f..6dcb0b1 100644 --- a/src/post/mod.rs +++ b/src/post/mod.rs @@ -1,54 +1,14 @@ pub mod cache; +pub mod markdown_posts; -use std::collections::BTreeSet; -use std::io; -use std::path::{Path, PathBuf}; -use std::time::{Duration, Instant, SystemTime}; +use std::time::Duration; +use axum::http::HeaderValue; use chrono::{DateTime, Utc}; -use fronma::parser::{parse, ParsedData}; use serde::{Deserialize, Serialize}; -use tokio::fs; -use tokio::io::AsyncReadExt; -use tracing::warn; -use crate::config::RenderConfig; -use crate::markdown_render::render; -use crate::post::cache::Cache; -use crate::systemtime_as_secs::as_secs; -use crate::PostError; - -#[derive(Deserialize)] -struct FrontMatter { - pub title: String, - pub description: String, - pub author: String, - pub icon: Option, - pub created_at: Option>, - pub modified_at: Option>, - #[serde(default)] - pub tags: BTreeSet, -} - -impl FrontMatter { - pub fn into_full( - self, - name: String, - created: Option, - modified: Option, - ) -> PostMetadata { - PostMetadata { - name, - title: self.title, - description: self.description, - author: self.author, - icon: self.icon, - created_at: self.created_at.or_else(|| created.map(|t| t.into())), - modified_at: self.modified_at.or_else(|| modified.map(|t| t.into())), - tags: self.tags.into_iter().collect(), - } - } -} +use crate::error::PostError; +pub use crate::post::markdown_posts::MarkdownPosts; #[derive(Serialize, Deserialize, Clone, Debug)] pub struct PostMetadata { @@ -62,168 +22,40 @@ pub struct PostMetadata { pub tags: Vec, } -#[allow(unused)] pub enum RenderStats { Cached(Duration), // format: Total, Parsed in, Rendered in ParsedAndRendered(Duration, Duration, Duration), } -#[derive(Clone)] -pub struct PostManager { - dir: PathBuf, - cache: Option, - config: RenderConfig, +#[allow(clippy::large_enum_variant)] // Raw will be returned very rarely +pub enum ReturnedPost { + Rendered(PostMetadata, String, RenderStats), + Raw(Vec, HeaderValue), } -impl PostManager { - pub fn new(dir: PathBuf, config: RenderConfig) -> PostManager { - PostManager { - dir, - cache: None, - config, - } - } - - pub fn new_with_cache(dir: PathBuf, config: RenderConfig, cache: Cache) -> PostManager { - PostManager { - dir, - cache: Some(cache), - config, - } - } - - async fn parse_and_render( - &self, - name: String, - path: impl AsRef, - ) -> Result<(PostMetadata, String, (Duration, Duration)), PostError> { - let parsing_start = Instant::now(); - let mut file = match tokio::fs::OpenOptions::new().read(true).open(&path).await { - Ok(val) => val, - Err(err) => match err.kind() { - io::ErrorKind::NotFound => return Err(PostError::NotFound(name)), - _ => return Err(PostError::IoError(err)), - }, - }; - let stat = file.metadata().await?; - let modified = stat.modified()?; - let created = stat.created().ok(); - - let mut content = String::with_capacity(stat.len() as usize); - file.read_to_string(&mut content).await?; - - let ParsedData { headers, body } = parse::(&content)?; - let metadata = headers.into_full(name.to_owned(), created, Some(modified)); - let parsing = parsing_start.elapsed(); - - let before_render = Instant::now(); - let post = render(body, &self.config); - let rendering = before_render.elapsed(); - - if let Some(cache) = self.cache.as_ref() { - cache - .insert( - name.to_string(), - metadata.clone(), - as_secs(&modified), - post.clone(), - &self.config, - ) - .await - .unwrap_or_else(|err| warn!("failed to insert {:?} into cache", err.0)) - }; - - Ok((metadata, post, (parsing, rendering))) - } - - pub async fn get_all_post_metadata_filtered( +pub trait PostManager { + async fn get_all_post_metadata( &self, filter: impl Fn(&PostMetadata) -> bool, ) -> Result, PostError> { - let mut posts = Vec::new(); - - let mut read_dir = fs::read_dir(&self.dir).await?; - while let Some(entry) = read_dir.next_entry().await? { - let path = entry.path(); - let stat = fs::metadata(&path).await?; - - if stat.is_file() && path.extension().is_some_and(|ext| ext == "md") { - let mtime = as_secs(&stat.modified()?); - // TODO. this? - let name = path - .clone() - .file_stem() - .unwrap() - .to_string_lossy() - .to_string(); - - if let Some(cache) = self.cache.as_ref() - && let Some(hit) = cache.lookup_metadata(&name, mtime).await - && filter(&hit) - { - posts.push(hit); - } else { - match self.parse_and_render(name, path).await { - Ok((metadata, ..)) => { - if filter(&metadata) { - posts.push(metadata); - } - } - Err(err) => match err { - PostError::IoError(ref io_err) - if matches!(io_err.kind(), io::ErrorKind::NotFound) => - { - warn!("TOCTOU: {}", err) - } - _ => return Err(err), - }, - } - } - } - } - - Ok(posts) + self.get_all_posts(|m, _| filter(m)) + .await + .map(|vec| vec.into_iter().map(|(meta, ..)| meta).collect()) } - pub async fn get_all_posts_filtered( + async fn get_all_posts( &self, filter: impl Fn(&PostMetadata, &str) -> bool, - ) -> Result, PostError> { - let mut posts = Vec::new(); + ) -> Result, PostError>; - let mut read_dir = fs::read_dir(&self.dir).await?; - while let Some(entry) = read_dir.next_entry().await? { - let path = entry.path(); - let stat = fs::metadata(&path).await?; - - if stat.is_file() && path.extension().is_some_and(|ext| ext == "md") { - let name = path - .clone() - .file_stem() - .unwrap() - .to_string_lossy() - .to_string(); - - let post = self.get_post(&name).await?; - if filter(&post.0, &post.1) { - posts.push(post); - } - } - } - - Ok(posts) - } - - pub async fn get_max_n_post_metadata_with_optional_tag_sorted( + async fn get_max_n_post_metadata_with_optional_tag_sorted( &self, n: Option, tag: Option<&String>, ) -> Result, PostError> { let mut posts = self - .get_all_post_metadata_filtered(|metadata| { - !tag.is_some_and(|tag| !metadata.tags.contains(tag)) - }) + .get_all_post_metadata(|metadata| !tag.is_some_and(|tag| !metadata.tags.contains(tag))) .await?; // we still want some semblance of order if created_at is None so sort by mtime as well posts.sort_unstable_by_key(|metadata| metadata.modified_at.unwrap_or_default()); @@ -236,59 +68,15 @@ impl PostManager { Ok(posts) } - pub async fn get_post( - &self, - name: &str, - ) -> Result<(PostMetadata, String, RenderStats), PostError> { - let start = Instant::now(); - let path = self.dir.join(name.to_owned() + ".md"); - - let stat = match tokio::fs::metadata(&path).await { - Ok(value) => value, - Err(err) => match err.kind() { - io::ErrorKind::NotFound => { - if let Some(cache) = self.cache.as_ref() { - cache.remove(name).await; - } - return Err(PostError::NotFound(name.to_string())); - } - _ => return Err(PostError::IoError(err)), - }, - }; - let mtime = as_secs(&stat.modified()?); - - if let Some(cache) = self.cache.as_ref() - && let Some(hit) = cache.lookup(name, mtime, &self.config).await - { - Ok(( - hit.metadata, - hit.rendered, - RenderStats::Cached(start.elapsed()), - )) - } else { - let (metadata, rendered, stats) = self.parse_and_render(name.to_string(), path).await?; - Ok(( - metadata, - rendered, - RenderStats::ParsedAndRendered(start.elapsed(), stats.0, stats.1), - )) + #[allow(unused)] + async fn get_post_metadata(&self, name: &str) -> Result { + match self.get_post(name).await? { + ReturnedPost::Rendered(metadata, ..) => Ok(metadata), + ReturnedPost::Raw(..) => Err(PostError::NotFound(name.to_string())), } } - pub fn cache(&self) -> Option<&Cache> { - self.cache.as_ref() - } + async fn get_post(&self, name: &str) -> Result; - pub async fn cleanup(&self) { - if let Some(cache) = self.cache.as_ref() { - cache - .cleanup(|name| { - std::fs::metadata(self.dir.join(name.to_owned() + ".md")) - .ok() - .and_then(|metadata| metadata.modified().ok()) - .map(|mtime| as_secs(&mtime)) - }) - .await - } - } + async fn cleanup(&self); } diff --git a/static/main.js b/static/main.js new file mode 100644 index 0000000..ad261ed --- /dev/null +++ b/static/main.js @@ -0,0 +1,4 @@ +for (let el of document.querySelectorAll(".date-rfc3339")) { + let date = new Date(Date.parse(el.textContent)); + el.textContent = date.toLocaleString(); +} diff --git a/static/style.css b/static/style.css index 21633a7..ac679b2 100644 --- a/static/style.css +++ b/static/style.css @@ -84,6 +84,35 @@ div.post { margin-bottom: 1em; } +.table { + display: grid; + /*grid-template-columns: auto auto auto; + grid-template-rows: auto auto;*/ + width: max-content; +} + +.table > :not(.value)::after { + content: ":"; +} + +.table > .value { + margin-left: 1em; + text-align: end; + grid-column: 2; +} + +.table > .created { + grid-row: 1; +} + +.table > .modified { + grid-row: 2; +} + +.table > .tags { + grid-row: 3; +} + /* BEGIN cool effect everyone liked */ body { diff --git a/templates/index.html b/templates/index.html index ee96b71..7e6291a 100644 --- a/templates/index.html +++ b/templates/index.html @@ -11,6 +11,8 @@ {% if rss %} + {% endif %} {% if js %} + {% endif %} diff --git a/templates/macros.askama b/templates/macros.askama index 17be7a6..bd21a25 100644 --- a/templates/macros.askama +++ b/templates/macros.askama @@ -1,19 +1,31 @@ +{% macro span_date(value) %} +{{ value|date(df) }} +{% endmacro %} {% macro table(post) %} +
{% match post.created_at %} {% when Some(created_at) %} - written:       {{ created_at|date }}
+
written
+
{% call span_date(created_at) %}
{% when None %} {% endmatch %} {% match post.modified_at %} {% when Some(modified_at) %} - last modified: {{ modified_at|date }}
+
last modified
+
{% call span_date(modified_at) %}
{% when None %} {% endmatch %} - {% if !post.tags.is_empty() %} - tags:          +
tags
+
{% for tag in post.tags %} {{ tag }} - {% endfor %}
+ {% endfor %} +
{% endif %} +
{% endmacro %} diff --git a/templates/post.html b/templates/post.html index a4ddf62..ba78574 100644 --- a/templates/post.html +++ b/templates/post.html @@ -2,20 +2,21 @@ - - - - - - - {% match meta.icon %} {% when Some with (url) %} - - - {% when None %} {% endmatch %} - {{ meta.title }} - - - + + + + + + {% match meta.icon %} {% when Some with (url) %} + + + {% when None %} {% endmatch %} + {{ meta.title }} + + + {% if js %} + + {% endif %}
@@ -24,11 +25,9 @@ - by {{ meta.author }}

{{ meta.description }}

-
+
-
{% call macros::table(meta) %} -
link
back to home