fix: update fetch logic to skip jobs fetched within the last 24 hours and adjust retry attempts in scraper

web/craigslist.py

@@ -175,10 +175,10 @@ def fetch_listings():
 def process_job_url(job_url: str, region: str = "", keyword: str = ""):
     last = get_last_fetch_time(job_url)
     if last is not None:
-        # skip if fetched within the last hour
+        # skip if fetched within the last 24 hours
         age = datetime.now(
             timezone.utc) - (last if last.tzinfo is not None else last.replace(tzinfo=timezone.utc))
-        if age.total_seconds() < 1 * 3600:
+        if age.total_seconds() < 24 * 3600:
             yield f"Skipping job {job_url} (fetched {age.seconds//3600}h ago)...\n"
             return None
 

web/scraper.py

@@ -224,7 +224,7 @@ def scrape_job_data(content: str, region: str, keyword: str, seen_urls: Set[str]
 def process_region_keyword(region: str, keyword: str, seen_urls: Set[str]) -> List[List]:
     """Process a single region and keyword."""
     url = get_base_url().format(region=region, keyword=keyword.replace(" ", "+"))
-    content = make_request_with_retry(url, 3)
+    content = make_request_with_retry(url, 1)
     if content is None:
         return []
     return scrape_job_data(content, region, keyword, seen_urls)