fix: update scraper settings and modify scrape form to use POST method
Some checks failed
CI/CD Pipeline / test (push) Successful in 1m32s
CI/CD Pipeline / build-image (push) Failing after 1m41s

This commit is contained in:
2026-01-07 16:25:35 +01:00
parent f8a5b1b5ef
commit 54ed111078
5 changed files with 60 additions and 25 deletions

View File

@@ -233,21 +233,6 @@ def scraper():
except StopIteration as stop:
listing_summary = stop.value if isinstance(stop.value, dict) else {}
new_jobs = []
if listing_summary:
new_jobs = listing_summary.get("new_jobs", []) or []
if new_jobs:
yield f"Preparing email alert for {len(new_jobs)} new jobs...\n"
try:
sent, info = _send_new_job_alert(new_jobs)
if sent:
yield "Job alert email sent.\n"
else:
yield f"Skipping email alert: {info}\n"
except Exception as exc:
yield f"Failed to send job alert email: {exc}\n"
# Finally, fetch and refresh individual job pages for current listings
job_urls = db_get_all_job_urls()
yield f"Processing {len(job_urls)} job pages...\n"
@@ -264,6 +249,21 @@ def scraper():
yield "\nScraping completed successfully!\n"
new_jobs = []
if listing_summary:
new_jobs = listing_summary.get("new_jobs", []) or []
if new_jobs:
yield f"Preparing email alert for {len(new_jobs)} new jobs...\n"
try:
sent, info = _send_new_job_alert(new_jobs)
if sent:
yield "Job alert email sent.\n"
else:
yield f"Skipping email alert: {info}\n"
except Exception as exc:
yield f"Failed to send job alert email: {exc}\n"
def scrape_jobs_with_retry(max_retries=3):
"""Run the scraping process with retry logic for failures."""