Some tweaks

.sqlx/query-55e409553723d53a32bd3fa0ae3e83976eb5d3ce1ab7ec2963d5e1a5525dfe6f.json

@@ -0,0 +1,26 @@
+{
+  "db_name": "SQLite",
+  "query": "SELECT url, last_checked_time as \"last_checked_time: chrono::DateTime<chrono::Utc>\" FROM feeds WHERE feed_id = ? AND user_id = ?",
+  "describe": {
+    "columns": [
+      {
+        "name": "url",
+        "ordinal": 0,
+        "type_info": "Text"
+      },
+      {
+        "name": "last_checked_time: chrono::DateTime<chrono::Utc>",
+        "ordinal": 1,
+        "type_info": "Datetime"
+      }
+    ],
+    "parameters": {
+      "Right": 2
+    },
+    "nullable": [
+      false,
+      false
+    ]
+  },
+  "hash": "55e409553723d53a32bd3fa0ae3e83976eb5d3ce1ab7ec2963d5e1a5525dfe6f"
+}

.sqlx/query-5ca2526f1ec4a055bc36898ef13a7d830df848285f4f6d66bb104935844b7f2d.json

@@ -1,20 +0,0 @@
-{
-  "db_name": "SQLite",
-  "query": "SELECT url FROM feeds WHERE feed_id = ? AND user_id = ?",
-  "describe": {
-    "columns": [
-      {
-        "name": "url",
-        "ordinal": 0,
-        "type_info": "Text"
-      }
-    ],
-    "parameters": {
-      "Right": 2
-    },
-    "nullable": [
-      false
-    ]
-  },
-  "hash": "5ca2526f1ec4a055bc36898ef13a7d830df848285f4f6d66bb104935844b7f2d"
-}

migrations/20240320000005_create_feed_entries.sql

@@ -0,0 +1,13 @@
+CREATE TABLE feed_entries (
+    id TEXT PRIMARY KEY,
+    feed_id TEXT NOT NULL,
+    title TEXT NOT NULL,
+    published TIMESTAMP,
+    updated TIMESTAMP,
+    summary TEXT NOT NULL,
+    content TEXT,
+    link TEXT,
+    created_at TIMESTAMP NOT NULL,
+    FOREIGN KEY (feed_id) REFERENCES feeds(feed_id) ON DELETE CASCADE,
+    UNIQUE(feed_id, id)
+); 

src/demo.rs

@@ -32,7 +32,7 @@ const DEMO_FEEDS: [DemoFeed; 5] = [
         name: "Astronomy Picture of the Day (APOD)",
         url: "https://apod.nasa.gov/apod.rss",
         category: None,
-    }
+    },
 ];
 
 pub async fn setup_demo_data(pool: &sqlx::SqlitePool) {

src/feed_utils.rs

@@ -5,6 +5,8 @@ use url::Url;
 pub struct FeedError;
 
 pub async fn fetch_feed(url: &Url) -> Result<feed_rs::model::Feed, FeedError> {
+    println!("Making a request to fetch feed `{url}`");
+
     // Fetch the feed content
     let response = reqwest::get(url.as_ref()).await.map_err(|e| {
         eprintln!("Failed to fetch feed: {}", e);

src/feeds.rs

@@ -31,7 +31,7 @@ impl Feed {
             url,
             user_id,
             added_time: now,
-            last_checked_time: now,
+            last_checked_time: chrono::DateTime::UNIX_EPOCH,
             categorization: Vec::new(),
         }
     }

src/poll.rs

@@ -1,11 +1,14 @@
 use crate::user::AuthenticatedUser;
 use crate::{feed_utils::fetch_feed, Db};
-use chrono::{DateTime, Utc};
+use chrono::{DateTime, Duration, Utc};
 use feed_rs::model::Text;
 use rocket::http::Status;
 use rocket::serde::uuid::Uuid;
-use rocket::serde::{json::Json, Serialize};
+use rocket::serde::{self, json::Json, Serialize};
 use rocket_db_pools::Connection;
+use sqlx::{Acquire, SqliteConnection};
+
+const POLLING_INTERVAL: Duration = Duration::minutes(20);
 
 #[derive(Debug, Serialize)]
 #[serde(crate = "rocket::serde")]
@@ -26,6 +29,66 @@ struct Entry {
     link: Option<String>,
 }
 
+async fn update_entry_db(entries: &Vec<Entry>, feed_id: &str, db: &mut SqliteConnection) -> Result<(), Status> {
+    // Start a transaction for batch update
+    let mut tx = db.begin().await.map_err(|e| {
+        eprintln!("Failed to start transaction: {}", e);
+        Status::InternalServerError
+    })?;
+
+    let now = Utc::now().to_rfc3339();
+    for entry in entries {
+        let content_json = if let Some(content) = &entry.content {
+            serde::json::to_string(content).ok()
+        } else {
+            None
+        };
+
+        let result = sqlx::query(
+            r#"
+            INSERT INTO feed_entries (
+                id, feed_id, title, published, updated, summary, content, link, created_at
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON CONFLICT (feed_id, id) DO UPDATE SET
+                title = excluded.title,
+                published = excluded.published,
+                updated = excluded.updated,
+                summary = excluded.summary,
+                content = excluded.content,
+                link = excluded.link
+            "#,
+        )
+        .bind(&entry.id)
+        .bind(feed_id)
+        .bind(&entry.title)
+        .bind(entry.published.map(|dt| dt.to_rfc3339()))
+        .bind(entry.updated.map(|dt| dt.to_rfc3339()))
+        .bind(&entry.summary)
+        .bind(content_json)
+        .bind(&entry.link)
+        .bind(&now)
+        .execute(&mut *tx)
+        .await;
+
+        if let Err(e) = result {
+            eprintln!("Failed to save feed entry: {}", e);
+            tx.rollback().await.map_err(|e| {
+                eprintln!("Failed to rollback transaction: {}", e);
+                Status::InternalServerError
+            })?;
+            return Err(Status::InternalServerError);
+        }
+    }
+
+    // Commit the transaction
+    tx.commit().await.map_err(|e| {
+        eprintln!("Failed to commit transaction: {}", e);
+        Status::InternalServerError
+    })?;
+
+    Ok(())
+}
+
 #[post("/poll/<feed_id>")]
 pub async fn poll_feed(
     mut db: Connection<Db>,
@@ -35,19 +98,26 @@ pub async fn poll_feed(
     let feed_id = feed_id.to_string();
     let user_id = user.user_id.to_string();
     // Get the feed URL from the database, ensuring it belongs to the authenticated user
-    let feed_url = sqlx::query!(
+    let feed = sqlx::query!(
-        "SELECT url FROM feeds WHERE feed_id = ? AND user_id = ?",
+        r#"SELECT url, last_checked_time as "last_checked_time: chrono::DateTime<chrono::Utc>" FROM feeds WHERE feed_id = ? AND user_id = ?"#,
         feed_id,
         user_id
     )
     .fetch_optional(&mut **db)
     .await
     .map_err(|_| Status::InternalServerError)?
-    .ok_or(Status::NotFound)?
+    .ok_or(Status::NotFound)?;
-    .url;
+
+    let now = Utc::now();
+    if now - feed.last_checked_time < POLLING_INTERVAL {
+        println!(
+            "Feed {} was checked recently at {}",
+            feed_id, feed.last_checked_time
+        );
+    }
 
     // Parse the URL
-    let url = url::Url::parse(&feed_url).map_err(|_| Status::InternalServerError)?;
+    let url = url::Url::parse(&feed.url).map_err(|_| Status::InternalServerError)?;
 
     let feed_data = fetch_feed(&url).await.map_err(|_| Status::BadGateway)?;
     let count = feed_data.entries.len();
@@ -57,7 +127,7 @@ pub async fn poll_feed(
             .unwrap_or(format!("<no {name}>"))
     }
 
-    let entries = feed_data
+    let entries: Vec<Entry> = feed_data
         .entries
         .into_iter()
         .map(|feed_entry| Entry {
@@ -71,5 +141,7 @@ pub async fn poll_feed(
         })
         .collect();
 
+    update_entry_db(&entries, &feed_id, &mut db).await?;
+
     Ok(Json(FeedPollResponse { count, entries }))
 }

src/session_store.rs

@@ -1,7 +1,7 @@
+use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
 use std::collections::{HashMap, HashSet};
 use std::sync::RwLock;
 use uuid::Uuid;
-use base64::{engine::general_purpose::STANDARD as BASE64, Engine as _};
 
 pub struct SessionStore(RwLock<HashMap<Uuid, HashSet<String>>>);
 
@@ -41,4 +41,4 @@ impl SessionStore {
             }
         }
     }
-} 
+}

src/user.rs

@@ -7,8 +7,8 @@ use rocket_db_pools::Connection;
 use rocket_dyn_templates::{context, Template};
 use uuid::Uuid;
 
-use crate::Db;
 use crate::session_store::SessionStore;
+use crate::Db;
 
 #[derive(Debug, Serialize)]
 #[serde(crate = "rocket::serde")]