updating db logic adding logging

web/db.py

@@ -147,6 +147,15 @@ class UserKeyword(Base):
         "keywords.keyword_id", ondelete="CASCADE"), primary_key=True)
 
 
+class Log(Base):
+    __tablename__ = "logs"
+    id = Column(Integer, primary_key=True, autoincrement=True)
+    page_url = Column(String(URL_LEN))
+    region = Column(String(SHORT_LEN))
+    keyword = Column(String(SHORT_LEN))
+    fetched_at = Column(DateTime)
+
+
 def _ensure_session() -> Session:
     global engine, SessionLocal
     if engine is None or SessionLocal is None:
@@ -218,7 +227,7 @@ def upsert_user_interaction(job_id: str | int, *, user_id: Optional[int] = None,
         session.commit()
 
 
-def upsert_listing(*, url: str, region: str, keyword: str, title: str, pay: str, location: str, timestamp: str):
+def upsert_listing(*, url: str, region: str, keyword: str, title: str, pay: str, location: str, timestamp: str, fetched_from: str | None = None, fetched_at: Optional[datetime] = None):
     """Insert or update a job listing row based on job_id derived from URL."""
     job_id = str(url_to_job_id(url))
     with _ensure_session() as session:
@@ -234,6 +243,33 @@ def upsert_listing(*, url: str, region: str, keyword: str, title: str, pay: str,
         setattr(obj, "location", location)
         setattr(obj, "timestamp", timestamp)
         session.commit()
+    # Optionally record a fetch log for the listing if source provided
+    if fetched_from:
+        try:
+            insert_log(fetched_from, region=region, keyword=keyword,
+                       fetched_at=fetched_at or datetime.now())
+        except Exception:
+            pass
+
+
+def insert_log(page_url: str, region: str | None = None, keyword: str | None = None, fetched_at: Optional[datetime] = None):
+    """Insert a log row for a fetched page."""
+    fetched_at = fetched_at or datetime.now()
+    with _ensure_session() as session:
+        l = Log(page_url=page_url, region=region or '',
+                keyword=keyword or '', fetched_at=fetched_at)
+        session.add(l)
+        session.commit()
+
+
+def get_last_fetch_time(page_url: str) -> Optional[datetime]:
+    """Return the latest fetched_at for a given page_url, or None if never fetched."""
+    with _ensure_session() as session:
+        row = session.execute(text("SELECT fetched_at FROM logs WHERE page_url = :u ORDER BY fetched_at DESC LIMIT 1"), {
+                              "u": page_url}).fetchone()
+        if row and row[0]:
+            return row[0]
+        return None
 
 
 def upsert_job_details(job_data: Dict[str, Any]):