fix: update fetch logic to skip jobs fetched within the last 24 hours and adjust retry attempts in scraper
This commit is contained in:
@@ -175,10 +175,10 @@ def fetch_listings():
|
||||
def process_job_url(job_url: str, region: str = "", keyword: str = ""):
|
||||
last = get_last_fetch_time(job_url)
|
||||
if last is not None:
|
||||
# skip if fetched within the last hour
|
||||
# skip if fetched within the last 24 hours
|
||||
age = datetime.now(
|
||||
timezone.utc) - (last if last.tzinfo is not None else last.replace(tzinfo=timezone.utc))
|
||||
if age.total_seconds() < 1 * 3600:
|
||||
if age.total_seconds() < 24 * 3600:
|
||||
yield f"Skipping job {job_url} (fetched {age.seconds//3600}h ago)...\n"
|
||||
return None
|
||||
|
||||
|
||||
Reference in New Issue
Block a user