Fixing feed entries

.sqlx/query-3c8d358e534f35c6e59b1d94cf28175b5d9b60a662388dec1c20a79e298cac4f.json

@@ -0,0 +1,12 @@
+{
+  "db_name": "SQLite",
+  "query": "\n            INSERT INTO feed_entries (\n                id, feed_id, entry_id, title, published, updated, summary, content, link, created_at\n            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)\n            ON CONFLICT (feed_id, id) DO UPDATE SET\n                title = excluded.title,\n                published = excluded.published,\n                updated = excluded.updated,\n                summary = excluded.summary,\n                content = excluded.content,\n                link = excluded.link\n            ",
+  "describe": {
+    "columns": [],
+    "parameters": {
+      "Right": 10
+    },
+    "nullable": []
+  },
+  "hash": "3c8d358e534f35c6e59b1d94cf28175b5d9b60a662388dec1c20a79e298cac4f"
+}

.sqlx/query-aa1176c799d37727714cf346eb8a16ba561dc7a1b9b95992c38248c2102d6621.json

@@ -0,0 +1,62 @@
+{
+  "db_name": "SQLite",
+  "query": "\n        SELECT \n            id as \"id!: String\",\n            entry_id,\n            title,\n            published as \"published: Option<chrono::DateTime<Utc>>\",\n            updated as \"updated: Option<chrono::DateTime<Utc>>\",\n            summary,\n            content,\n            link\n        FROM feed_entries \n        WHERE feed_id = ?\n        ORDER BY published DESC NULLS LAST\n        LIMIT ?\n        ",
+  "describe": {
+    "columns": [
+      {
+        "name": "id!: String",
+        "ordinal": 0,
+        "type_info": "Text"
+      },
+      {
+        "name": "entry_id",
+        "ordinal": 1,
+        "type_info": "Text"
+      },
+      {
+        "name": "title",
+        "ordinal": 2,
+        "type_info": "Text"
+      },
+      {
+        "name": "published: Option<chrono::DateTime<Utc>>",
+        "ordinal": 3,
+        "type_info": "Datetime"
+      },
+      {
+        "name": "updated: Option<chrono::DateTime<Utc>>",
+        "ordinal": 4,
+        "type_info": "Datetime"
+      },
+      {
+        "name": "summary",
+        "ordinal": 5,
+        "type_info": "Text"
+      },
+      {
+        "name": "content",
+        "ordinal": 6,
+        "type_info": "Text"
+      },
+      {
+        "name": "link",
+        "ordinal": 7,
+        "type_info": "Text"
+      }
+    ],
+    "parameters": {
+      "Right": 2
+    },
+    "nullable": [
+      true,
+      false,
+      false,
+      true,
+      true,
+      false,
+      true,
+      true
+    ]
+  },
+  "hash": "aa1176c799d37727714cf346eb8a16ba561dc7a1b9b95992c38248c2102d6621"
+}

TODO.md

@@ -17,3 +17,8 @@ Current session management is basic and needs improvement:
   - Include only the icons we actually use to reduce bundle size
   - Consider using Font Awesome's SVG+JS version for better performance
   - Update CSS and HTML references to use local assets
+
+- [ ] Add a timeout to external RSS feed fetching to prevent hanging on slow feeds
+  - Use reqwest's timeout feature
+  - Consider making the timeout configurable
+  - Add error handling for timeout cases

migrations/20240320000006_add_marked_read.sql

@@ -0,0 +1 @@
+ALTER TABLE feed_entries ADD COLUMN marked_read TIMESTAMP DEFAULT NULL; 

src/feed_utils.rs

@@ -24,9 +24,6 @@ pub async fn fetch_feed(url: &Url) -> Result<feed_rs::model::Feed, FeedError> {
     })?;
 
     println!("Fetched feed: {}", url.as_ref());
-    for item in &feed_data.entries {
-        println!("{:?}", item);
-    }
 
     Ok(feed_data)
 }

src/poll.rs

@@ -10,6 +10,7 @@ use sqlx::{Acquire, SqliteConnection};
 use url::Url;
 
 const POLLING_INTERVAL: Duration = Duration::minutes(20);
+const MAX_ENTRIES_PER_FEED: i32 = 30;
 
 #[derive(Debug, Serialize)]
 #[serde(crate = "rocket::serde")]
@@ -43,6 +44,18 @@ async fn update_entry_db(
     })?;
 
     let now = Utc::now().to_rfc3339();
+
+    // Update the feed's last_checked_time
+    sqlx::query("UPDATE feeds SET last_checked_time = ? WHERE feed_id = ?")
+        .bind(&now)
+        .bind(feed_id)
+        .execute(&mut *tx)
+        .await
+        .map_err(|e| {
+            eprintln!("Failed to update feed last_checked_time: {}", e);
+            Status::InternalServerError
+        })?;
+
     for entry in entries {
         let content_json = if let Some(content) = &entry.content {
             serde::json::to_string(content).ok()
@@ -50,11 +63,15 @@ async fn update_entry_db(
             None
         };
 
-        let result = sqlx::query(
+        let published = entry.published.map(|dt| dt.to_rfc3339());
+        let updated = entry.updated.map(|dt| dt.to_rfc3339());
+
+        let entry_id = entry.id.to_string();
+        let result = sqlx::query!(
             r#"
             INSERT INTO feed_entries (
                 id, feed_id, entry_id, title, published, updated, summary, content, link, created_at
-            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
             ON CONFLICT (feed_id, id) DO UPDATE SET
                 title = excluded.title,
                 published = excluded.published,
@@ -63,17 +80,17 @@ async fn update_entry_db(
                 content = excluded.content,
                 link = excluded.link
             "#,
+            entry_id,
+            feed_id,
+            entry.entry_id,
+            entry.title,
+            published,
+            updated,
+            entry.summary,
+            content_json,
+            entry.link,
+            now
         )
-        .bind(&entry.id)
-        .bind(feed_id)
-        .bind(&entry.entry_id)
-        .bind(&entry.title)
-        .bind(entry.published.map(|dt| dt.to_rfc3339()))
-        .bind(entry.updated.map(|dt| dt.to_rfc3339()))
-        .bind(&entry.summary)
-        .bind(content_json)
-        .bind(&entry.link)
-        .bind(&now)
         .execute(&mut *tx)
         .await;
 
@@ -96,6 +113,57 @@ async fn update_entry_db(
     Ok(())
 }
 
+async fn read_entries(feed_id: &str, db: &mut SqliteConnection) -> Result<Vec<Entry>, Status> {
+    let rows = sqlx::query!(
+        r#"
+        SELECT 
+            id as "id!: String",
+            entry_id,
+            title,
+            published as "published: Option<chrono::DateTime<Utc>>",
+            updated as "updated: Option<chrono::DateTime<Utc>>",
+            summary,
+            content,
+            link
+        FROM feed_entries 
+        WHERE feed_id = ?
+        ORDER BY published DESC NULLS LAST
+        LIMIT ?
+        "#,
+        feed_id,
+        MAX_ENTRIES_PER_FEED,
+    )
+    .fetch_all(db)
+    .await
+    .map_err(|e| {
+        eprintln!("Failed to read feed entries: {}", e);
+        Status::InternalServerError
+    })?;
+
+    let entries = rows
+        .iter()
+        .map(|row| {
+            let content: Option<feed_rs::model::Content> = row
+                .content
+                .as_ref()
+                .and_then(|s| serde::json::from_str(s).ok());
+
+            Ok(Entry {
+                id: Uuid::parse_str(&row.id).map_err(|_| Status::InternalServerError)?,
+                entry_id: row.entry_id.clone(),
+                title: row.title.clone(),
+                published: row.published.flatten(),
+                updated: row.updated.flatten(),
+                summary: row.summary.clone(),
+                content,
+                link: row.link.clone(),
+            })
+        })
+        .collect::<Result<Vec<_>, Status>>()?;
+
+    Ok(entries)
+}
+
 /// Perform the request to fetch from the remote feed url
 async fn fetch_new_entries(url: &Url) -> Result<Vec<Entry>, Status> {
     let feed_data = fetch_feed(url).await.map_err(|_| Status::BadGateway)?;
@@ -145,16 +213,18 @@ pub async fn poll_feed(
     let url = url::Url::parse(&feed.url).map_err(|_| Status::InternalServerError)?;
 
     let now = Utc::now();
-    if now - feed.last_checked_time < POLLING_INTERVAL {
-        println!(
-            "Feed {} was checked recently at {}",
-            feed_id, feed.last_checked_time
-        );
-    }
+    let last_checked = now - feed.last_checked_time;
+    println!("Feed last checked: {}", last_checked);
+    let entries = if last_checked < POLLING_INTERVAL {
+        println!("reading entries out of db");
+        read_entries(&feed_id, &mut db).await?
+    } else {
+        let entries = fetch_new_entries(&url).await?;
+        update_entry_db(&entries, &feed_id, &mut db).await?;
+        entries
+    };
 
-    let entries = fetch_new_entries(&url).await?;
     let count = entries.len();
-    update_entry_db(&entries, &feed_id, &mut db).await?;
 
     Ok(Json(FeedPollResponse { count, entries }))
 }