fix: update fetch logic to skip jobs fetched within the last 24 hours and adjust retry attempts in scraper
Some checks failed
CI/CD Pipeline / test (push) Failing after 20s
CI/CD Pipeline / build-image (push) Has been skipped

This commit is contained in:
2025-11-28 20:54:39 +01:00
parent e0bc295936
commit 02e3e77f78
3 changed files with 4 additions and 3 deletions

View File

@@ -224,7 +224,7 @@ def scrape_job_data(content: str, region: str, keyword: str, seen_urls: Set[str]
def process_region_keyword(region: str, keyword: str, seen_urls: Set[str]) -> List[List]:
"""Process a single region and keyword."""
url = get_base_url().format(region=region, keyword=keyword.replace(" ", "+"))
content = make_request_with_retry(url, 3)
content = make_request_with_retry(url, 1)
if content is None:
return []
return scrape_job_data(content, region, keyword, seen_urls)