fix: update fetch logic to skip jobs fetched within the last 24 hours and adjust retry attempts in scraper
This commit is contained in:
@@ -224,7 +224,7 @@ def scrape_job_data(content: str, region: str, keyword: str, seen_urls: Set[str]
|
||||
def process_region_keyword(region: str, keyword: str, seen_urls: Set[str]) -> List[List]:
|
||||
"""Process a single region and keyword."""
|
||||
url = get_base_url().format(region=region, keyword=keyword.replace(" ", "+"))
|
||||
content = make_request_with_retry(url, 3)
|
||||
content = make_request_with_retry(url, 1)
|
||||
if content is None:
|
||||
return []
|
||||
return scrape_job_data(content, region, keyword, seen_urls)
|
||||
|
||||
Reference in New Issue
Block a user