From 6d58a7d70e1b8ac1910a077df22be76e1caf6327 Mon Sep 17 00:00:00 2001 From: Greg Shuflin Date: Tue, 4 Feb 2025 14:29:17 -0800 Subject: [PATCH] entry db --- ...e83976eb5d3ce1ab7ec2963d5e1a5525dfe6f.json | 26 ++++ ...a7d830df848285f4f6d66bb104935844b7f2d.json | 20 --- .../20240320000005_create_feed_entries.sql | 13 ++ src/demo.rs | 2 +- src/poll.rs | 115 ++++++++++-------- src/session_store.rs | 4 +- src/user.rs | 2 +- 7 files changed, 108 insertions(+), 74 deletions(-) create mode 100644 .sqlx/query-55e409553723d53a32bd3fa0ae3e83976eb5d3ce1ab7ec2963d5e1a5525dfe6f.json delete mode 100644 .sqlx/query-5ca2526f1ec4a055bc36898ef13a7d830df848285f4f6d66bb104935844b7f2d.json create mode 100644 migrations/20240320000005_create_feed_entries.sql diff --git a/.sqlx/query-55e409553723d53a32bd3fa0ae3e83976eb5d3ce1ab7ec2963d5e1a5525dfe6f.json b/.sqlx/query-55e409553723d53a32bd3fa0ae3e83976eb5d3ce1ab7ec2963d5e1a5525dfe6f.json new file mode 100644 index 0000000..7c3b659 --- /dev/null +++ b/.sqlx/query-55e409553723d53a32bd3fa0ae3e83976eb5d3ce1ab7ec2963d5e1a5525dfe6f.json @@ -0,0 +1,26 @@ +{ + "db_name": "SQLite", + "query": "SELECT url, last_checked_time as \"last_checked_time: chrono::DateTime\" FROM feeds WHERE feed_id = ? AND user_id = ?", + "describe": { + "columns": [ + { + "name": "url", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "last_checked_time: chrono::DateTime", + "ordinal": 1, + "type_info": "Datetime" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + false, + false + ] + }, + "hash": "55e409553723d53a32bd3fa0ae3e83976eb5d3ce1ab7ec2963d5e1a5525dfe6f" +} diff --git a/.sqlx/query-5ca2526f1ec4a055bc36898ef13a7d830df848285f4f6d66bb104935844b7f2d.json b/.sqlx/query-5ca2526f1ec4a055bc36898ef13a7d830df848285f4f6d66bb104935844b7f2d.json deleted file mode 100644 index 2a740ce..0000000 --- a/.sqlx/query-5ca2526f1ec4a055bc36898ef13a7d830df848285f4f6d66bb104935844b7f2d.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "db_name": "SQLite", - "query": "SELECT url FROM feeds WHERE feed_id = ? AND user_id = ?", - "describe": { - "columns": [ - { - "name": "url", - "ordinal": 0, - "type_info": "Text" - } - ], - "parameters": { - "Right": 2 - }, - "nullable": [ - false - ] - }, - "hash": "5ca2526f1ec4a055bc36898ef13a7d830df848285f4f6d66bb104935844b7f2d" -} diff --git a/migrations/20240320000005_create_feed_entries.sql b/migrations/20240320000005_create_feed_entries.sql new file mode 100644 index 0000000..dde90bb --- /dev/null +++ b/migrations/20240320000005_create_feed_entries.sql @@ -0,0 +1,13 @@ +CREATE TABLE feed_entries ( + id TEXT PRIMARY KEY, + feed_id TEXT NOT NULL, + title TEXT NOT NULL, + published TIMESTAMP, + updated TIMESTAMP, + summary TEXT NOT NULL, + content TEXT, + link TEXT, + created_at TIMESTAMP NOT NULL, + FOREIGN KEY (feed_id) REFERENCES feeds(feed_id) ON DELETE CASCADE, + UNIQUE(feed_id, id) +); \ No newline at end of file diff --git a/src/demo.rs b/src/demo.rs index c8a3b9d..2872314 100644 --- a/src/demo.rs +++ b/src/demo.rs @@ -32,7 +32,7 @@ const DEMO_FEEDS: [DemoFeed; 5] = [ name: "Astronomy Picture of the Day (APOD)", url: "https://apod.nasa.gov/apod.rss", category: None, - } + }, ]; pub async fn setup_demo_data(pool: &sqlx::SqlitePool) { diff --git a/src/poll.rs b/src/poll.rs index a5f409c..542edfe 100644 --- a/src/poll.rs +++ b/src/poll.rs @@ -1,12 +1,14 @@ use crate::user::AuthenticatedUser; use crate::{feed_utils::fetch_feed, Db}; -use chrono::{DateTime, Utc}; +use chrono::{DateTime, Duration, Utc}; use feed_rs::model::Text; use rocket::http::Status; use rocket::serde::uuid::Uuid; use rocket::serde::{self, json::Json, Serialize}; use rocket_db_pools::Connection; -use sqlx::Acquire; +use sqlx::{Acquire, SqliteConnection}; + +const POLLING_INTERVAL: Duration = Duration::minutes(20); #[derive(Debug, Serialize)] #[serde(crate = "rocket::serde")] @@ -27,51 +29,7 @@ struct Entry { link: Option, } -#[post("/poll/")] -pub async fn poll_feed( - mut db: Connection, - feed_id: Uuid, - user: AuthenticatedUser, -) -> Result, Status> { - let feed_id = feed_id.to_string(); - let user_id = user.user_id.to_string(); - // Get the feed URL from the database, ensuring it belongs to the authenticated user - let feed_url = sqlx::query!( - "SELECT url FROM feeds WHERE feed_id = ? AND user_id = ?", - feed_id, - user_id - ) - .fetch_optional(&mut **db) - .await - .map_err(|_| Status::InternalServerError)? - .ok_or(Status::NotFound)? - .url; - - // Parse the URL - let url = url::Url::parse(&feed_url).map_err(|_| Status::InternalServerError)?; - - let feed_data = fetch_feed(&url).await.map_err(|_| Status::BadGateway)?; - let count = feed_data.entries.len(); - - fn get(item: Option, name: &'static str) -> String { - item.map(|t| t.content.to_string()) - .unwrap_or(format!("")) - } - - let entries: Vec = feed_data - .entries - .into_iter() - .map(|feed_entry| Entry { - id: feed_entry.id, - title: get(feed_entry.title, "title"), - published: feed_entry.published, - updated: feed_entry.updated, - summary: get(feed_entry.summary, "summary"), - content: feed_entry.content, - link: feed_entry.links.first().map(|l| l.href.clone()), - }) - .collect(); - +async fn update_entry_db(entries: &Vec, feed_id: &str, db: &mut SqliteConnection) -> Result<(), Status> { // Start a transaction for batch update let mut tx = db.begin().await.map_err(|e| { eprintln!("Failed to start transaction: {}", e); @@ -79,7 +37,7 @@ pub async fn poll_feed( })?; let now = Utc::now().to_rfc3339(); - for entry in &entries { + for entry in entries { let content_json = if let Some(content) = &entry.content { serde::json::to_string(content).ok() } else { @@ -98,10 +56,10 @@ pub async fn poll_feed( summary = excluded.summary, content = excluded.content, link = excluded.link - "# + "#, ) .bind(&entry.id) - .bind(&feed_id) + .bind(feed_id) .bind(&entry.title) .bind(entry.published.map(|dt| dt.to_rfc3339())) .bind(entry.updated.map(|dt| dt.to_rfc3339())) @@ -128,5 +86,62 @@ pub async fn poll_feed( Status::InternalServerError })?; + Ok(()) +} + +#[post("/poll/")] +pub async fn poll_feed( + mut db: Connection, + feed_id: Uuid, + user: AuthenticatedUser, +) -> Result, Status> { + let feed_id = feed_id.to_string(); + let user_id = user.user_id.to_string(); + // Get the feed URL from the database, ensuring it belongs to the authenticated user + let feed = sqlx::query!( + r#"SELECT url, last_checked_time as "last_checked_time: chrono::DateTime" FROM feeds WHERE feed_id = ? AND user_id = ?"#, + feed_id, + user_id + ) + .fetch_optional(&mut **db) + .await + .map_err(|_| Status::InternalServerError)? + .ok_or(Status::NotFound)?; + + let now = Utc::now(); + if now - feed.last_checked_time < POLLING_INTERVAL { + println!( + "Feed {} was checked recently at {}", + feed_id, feed.last_checked_time + ); + } + + // Parse the URL + let url = url::Url::parse(&feed.url).map_err(|_| Status::InternalServerError)?; + + let feed_data = fetch_feed(&url).await.map_err(|_| Status::BadGateway)?; + let count = feed_data.entries.len(); + + fn get(item: Option, name: &'static str) -> String { + item.map(|t| t.content.to_string()) + .unwrap_or(format!("")) + } + + let entries: Vec = feed_data + .entries + .into_iter() + .map(|feed_entry| Entry { + id: feed_entry.id, + title: get(feed_entry.title, "title"), + published: feed_entry.published, + updated: feed_entry.updated, + summary: get(feed_entry.summary, "summary"), + content: feed_entry.content, + link: feed_entry.links.first().map(|l| l.href.clone()), + }) + .collect(); + + update_entry_db(&entries, &feed_id, &mut **db).await?; + Ok(Json(FeedPollResponse { count, entries })) } diff --git a/src/session_store.rs b/src/session_store.rs index 5f2ae99..65aec0d 100644 --- a/src/session_store.rs +++ b/src/session_store.rs @@ -1,7 +1,7 @@ +use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; use std::collections::{HashMap, HashSet}; use std::sync::RwLock; use uuid::Uuid; -use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _}; pub struct SessionStore(RwLock>>); @@ -41,4 +41,4 @@ impl SessionStore { } } } -} \ No newline at end of file +} diff --git a/src/user.rs b/src/user.rs index 990a4ec..dd4ccbf 100644 --- a/src/user.rs +++ b/src/user.rs @@ -7,8 +7,8 @@ use rocket_db_pools::Connection; use rocket_dyn_templates::{context, Template}; use uuid::Uuid; -use crate::Db; use crate::session_store::SessionStore; +use crate::Db; #[derive(Debug, Serialize)] #[serde(crate = "rocket::serde")]