From 69aa7393fe837f56de605ccbf2d97834c31e4832 Mon Sep 17 00:00:00 2001 From: Greg Shuflin Date: Tue, 4 Feb 2025 23:49:39 -0800 Subject: [PATCH] Fixing feed entries --- ...8175b5d9b60a662388dec1c20a79e298cac4f.json | 12 ++ ...a16ba561dc7a1b9b95992c38248c2102d6621.json | 62 ++++++++++ TODO.md | 5 + migrations/20240320000006_add_marked_read.sql | 1 + src/feed_utils.rs | 3 - src/poll.rs | 110 ++++++++++++++---- 6 files changed, 170 insertions(+), 23 deletions(-) create mode 100644 .sqlx/query-3c8d358e534f35c6e59b1d94cf28175b5d9b60a662388dec1c20a79e298cac4f.json create mode 100644 .sqlx/query-aa1176c799d37727714cf346eb8a16ba561dc7a1b9b95992c38248c2102d6621.json create mode 100644 migrations/20240320000006_add_marked_read.sql diff --git a/.sqlx/query-3c8d358e534f35c6e59b1d94cf28175b5d9b60a662388dec1c20a79e298cac4f.json b/.sqlx/query-3c8d358e534f35c6e59b1d94cf28175b5d9b60a662388dec1c20a79e298cac4f.json new file mode 100644 index 0000000..9759c8f --- /dev/null +++ b/.sqlx/query-3c8d358e534f35c6e59b1d94cf28175b5d9b60a662388dec1c20a79e298cac4f.json @@ -0,0 +1,12 @@ +{ + "db_name": "SQLite", + "query": "\n INSERT INTO feed_entries (\n id, feed_id, entry_id, title, published, updated, summary, content, link, created_at\n ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n ON CONFLICT (feed_id, id) DO UPDATE SET\n title = excluded.title,\n published = excluded.published,\n updated = excluded.updated,\n summary = excluded.summary,\n content = excluded.content,\n link = excluded.link\n ", + "describe": { + "columns": [], + "parameters": { + "Right": 10 + }, + "nullable": [] + }, + "hash": "3c8d358e534f35c6e59b1d94cf28175b5d9b60a662388dec1c20a79e298cac4f" +} diff --git a/.sqlx/query-aa1176c799d37727714cf346eb8a16ba561dc7a1b9b95992c38248c2102d6621.json b/.sqlx/query-aa1176c799d37727714cf346eb8a16ba561dc7a1b9b95992c38248c2102d6621.json new file mode 100644 index 0000000..b1bad20 --- /dev/null +++ b/.sqlx/query-aa1176c799d37727714cf346eb8a16ba561dc7a1b9b95992c38248c2102d6621.json @@ -0,0 +1,62 @@ +{ + "db_name": "SQLite", + "query": "\n SELECT \n id as \"id!: String\",\n entry_id,\n title,\n published as \"published: Option>\",\n updated as \"updated: Option>\",\n summary,\n content,\n link\n FROM feed_entries \n WHERE feed_id = ?\n ORDER BY published DESC NULLS LAST\n LIMIT ?\n ", + "describe": { + "columns": [ + { + "name": "id!: String", + "ordinal": 0, + "type_info": "Text" + }, + { + "name": "entry_id", + "ordinal": 1, + "type_info": "Text" + }, + { + "name": "title", + "ordinal": 2, + "type_info": "Text" + }, + { + "name": "published: Option>", + "ordinal": 3, + "type_info": "Datetime" + }, + { + "name": "updated: Option>", + "ordinal": 4, + "type_info": "Datetime" + }, + { + "name": "summary", + "ordinal": 5, + "type_info": "Text" + }, + { + "name": "content", + "ordinal": 6, + "type_info": "Text" + }, + { + "name": "link", + "ordinal": 7, + "type_info": "Text" + } + ], + "parameters": { + "Right": 2 + }, + "nullable": [ + true, + false, + false, + true, + true, + false, + true, + true + ] + }, + "hash": "aa1176c799d37727714cf346eb8a16ba561dc7a1b9b95992c38248c2102d6621" +} diff --git a/TODO.md b/TODO.md index 0fbf96b..b9852b1 100644 --- a/TODO.md +++ b/TODO.md @@ -17,3 +17,8 @@ Current session management is basic and needs improvement: - Include only the icons we actually use to reduce bundle size - Consider using Font Awesome's SVG+JS version for better performance - Update CSS and HTML references to use local assets + +- [ ] Add a timeout to external RSS feed fetching to prevent hanging on slow feeds + - Use reqwest's timeout feature + - Consider making the timeout configurable + - Add error handling for timeout cases diff --git a/migrations/20240320000006_add_marked_read.sql b/migrations/20240320000006_add_marked_read.sql new file mode 100644 index 0000000..d1a394f --- /dev/null +++ b/migrations/20240320000006_add_marked_read.sql @@ -0,0 +1 @@ +ALTER TABLE feed_entries ADD COLUMN marked_read TIMESTAMP DEFAULT NULL; \ No newline at end of file diff --git a/src/feed_utils.rs b/src/feed_utils.rs index e25b500..d37db18 100644 --- a/src/feed_utils.rs +++ b/src/feed_utils.rs @@ -24,9 +24,6 @@ pub async fn fetch_feed(url: &Url) -> Result { })?; println!("Fetched feed: {}", url.as_ref()); - for item in &feed_data.entries { - println!("{:?}", item); - } Ok(feed_data) } diff --git a/src/poll.rs b/src/poll.rs index 89e02c5..45e0e01 100644 --- a/src/poll.rs +++ b/src/poll.rs @@ -10,6 +10,7 @@ use sqlx::{Acquire, SqliteConnection}; use url::Url; const POLLING_INTERVAL: Duration = Duration::minutes(20); +const MAX_ENTRIES_PER_FEED: i32 = 30; #[derive(Debug, Serialize)] #[serde(crate = "rocket::serde")] @@ -43,6 +44,18 @@ async fn update_entry_db( })?; let now = Utc::now().to_rfc3339(); + + // Update the feed's last_checked_time + sqlx::query("UPDATE feeds SET last_checked_time = ? WHERE feed_id = ?") + .bind(&now) + .bind(feed_id) + .execute(&mut *tx) + .await + .map_err(|e| { + eprintln!("Failed to update feed last_checked_time: {}", e); + Status::InternalServerError + })?; + for entry in entries { let content_json = if let Some(content) = &entry.content { serde::json::to_string(content).ok() @@ -50,11 +63,15 @@ async fn update_entry_db( None }; - let result = sqlx::query( + let published = entry.published.map(|dt| dt.to_rfc3339()); + let updated = entry.updated.map(|dt| dt.to_rfc3339()); + + let entry_id = entry.id.to_string(); + let result = sqlx::query!( r#" INSERT INTO feed_entries ( id, feed_id, entry_id, title, published, updated, summary, content, link, created_at - ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (feed_id, id) DO UPDATE SET title = excluded.title, published = excluded.published, @@ -63,17 +80,17 @@ async fn update_entry_db( content = excluded.content, link = excluded.link "#, + entry_id, + feed_id, + entry.entry_id, + entry.title, + published, + updated, + entry.summary, + content_json, + entry.link, + now ) - .bind(&entry.id) - .bind(feed_id) - .bind(&entry.entry_id) - .bind(&entry.title) - .bind(entry.published.map(|dt| dt.to_rfc3339())) - .bind(entry.updated.map(|dt| dt.to_rfc3339())) - .bind(&entry.summary) - .bind(content_json) - .bind(&entry.link) - .bind(&now) .execute(&mut *tx) .await; @@ -96,6 +113,57 @@ async fn update_entry_db( Ok(()) } +async fn read_entries(feed_id: &str, db: &mut SqliteConnection) -> Result, Status> { + let rows = sqlx::query!( + r#" + SELECT + id as "id!: String", + entry_id, + title, + published as "published: Option>", + updated as "updated: Option>", + summary, + content, + link + FROM feed_entries + WHERE feed_id = ? + ORDER BY published DESC NULLS LAST + LIMIT ? + "#, + feed_id, + MAX_ENTRIES_PER_FEED, + ) + .fetch_all(db) + .await + .map_err(|e| { + eprintln!("Failed to read feed entries: {}", e); + Status::InternalServerError + })?; + + let entries = rows + .iter() + .map(|row| { + let content: Option = row + .content + .as_ref() + .and_then(|s| serde::json::from_str(s).ok()); + + Ok(Entry { + id: Uuid::parse_str(&row.id).map_err(|_| Status::InternalServerError)?, + entry_id: row.entry_id.clone(), + title: row.title.clone(), + published: row.published.flatten(), + updated: row.updated.flatten(), + summary: row.summary.clone(), + content, + link: row.link.clone(), + }) + }) + .collect::, Status>>()?; + + Ok(entries) +} + /// Perform the request to fetch from the remote feed url async fn fetch_new_entries(url: &Url) -> Result, Status> { let feed_data = fetch_feed(url).await.map_err(|_| Status::BadGateway)?; @@ -145,16 +213,18 @@ pub async fn poll_feed( let url = url::Url::parse(&feed.url).map_err(|_| Status::InternalServerError)?; let now = Utc::now(); - if now - feed.last_checked_time < POLLING_INTERVAL { - println!( - "Feed {} was checked recently at {}", - feed_id, feed.last_checked_time - ); - } + let last_checked = now - feed.last_checked_time; + println!("Feed last checked: {}", last_checked); + let entries = if last_checked < POLLING_INTERVAL { + println!("reading entries out of db"); + read_entries(&feed_id, &mut db).await? + } else { + let entries = fetch_new_entries(&url).await?; + update_entry_db(&entries, &feed_id, &mut db).await?; + entries + }; - let entries = fetch_new_entries(&url).await?; let count = entries.len(); - update_entry_db(&entries, &feed_id, &mut db).await?; Ok(Json(FeedPollResponse { count, entries })) }