fix: update scraper settings and modify scrape form to use POST method
This commit is contained in:
@@ -233,21 +233,6 @@ def scraper():
|
||||
except StopIteration as stop:
|
||||
listing_summary = stop.value if isinstance(stop.value, dict) else {}
|
||||
|
||||
new_jobs = []
|
||||
if listing_summary:
|
||||
new_jobs = listing_summary.get("new_jobs", []) or []
|
||||
|
||||
if new_jobs:
|
||||
yield f"Preparing email alert for {len(new_jobs)} new jobs...\n"
|
||||
try:
|
||||
sent, info = _send_new_job_alert(new_jobs)
|
||||
if sent:
|
||||
yield "Job alert email sent.\n"
|
||||
else:
|
||||
yield f"Skipping email alert: {info}\n"
|
||||
except Exception as exc:
|
||||
yield f"Failed to send job alert email: {exc}\n"
|
||||
|
||||
# Finally, fetch and refresh individual job pages for current listings
|
||||
job_urls = db_get_all_job_urls()
|
||||
yield f"Processing {len(job_urls)} job pages...\n"
|
||||
@@ -264,6 +249,21 @@ def scraper():
|
||||
|
||||
yield "\nScraping completed successfully!\n"
|
||||
|
||||
new_jobs = []
|
||||
if listing_summary:
|
||||
new_jobs = listing_summary.get("new_jobs", []) or []
|
||||
|
||||
if new_jobs:
|
||||
yield f"Preparing email alert for {len(new_jobs)} new jobs...\n"
|
||||
try:
|
||||
sent, info = _send_new_job_alert(new_jobs)
|
||||
if sent:
|
||||
yield "Job alert email sent.\n"
|
||||
else:
|
||||
yield f"Skipping email alert: {info}\n"
|
||||
except Exception as exc:
|
||||
yield f"Failed to send job alert email: {exc}\n"
|
||||
|
||||
|
||||
def scrape_jobs_with_retry(max_retries=3):
|
||||
"""Run the scraping process with retry logic for failures."""
|
||||
|
||||
Reference in New Issue
Block a user