fix: update fetch logic to skip jobs fetched within the last 24 hours and adjust retry attempts in scraper
Some checks failed
CI/CD Pipeline / test (push) Failing after 20s
CI/CD Pipeline / build-image (push) Has been skipped

This commit is contained in:
2025-11-28 20:54:39 +01:00
parent e0bc295936
commit 02e3e77f78
3 changed files with 4 additions and 3 deletions

View File

@@ -175,10 +175,10 @@ def fetch_listings():
def process_job_url(job_url: str, region: str = "", keyword: str = ""):
last = get_last_fetch_time(job_url)
if last is not None:
# skip if fetched within the last hour
# skip if fetched within the last 24 hours
age = datetime.now(
timezone.utc) - (last if last.tzinfo is not None else last.replace(tzinfo=timezone.utc))
if age.total_seconds() < 1 * 3600:
if age.total_seconds() < 24 * 3600:
yield f"Skipping job {job_url} (fetched {age.seconds//3600}h ago)...\n"
return None