do not remove stale urls
This commit is contained in:
@@ -94,19 +94,8 @@ def fetch_listings():
|
||||
fetched_at=datetime.now(timezone.utc),
|
||||
)
|
||||
|
||||
# Remove stale listings: those present in DB but not discovered now.
|
||||
stale_urls = existing_db_urls - discovered_urls
|
||||
if stale_urls:
|
||||
yield f"Removing {len(stale_urls)} stale listings...\n"
|
||||
for url in stale_urls:
|
||||
try:
|
||||
jid = url_to_job_id(url)
|
||||
db_delete_job(jid)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
yield f"Listing fetch complete: {len(discovered_urls)} discovered, {len(new_rows)} new, {len(stale_urls)} stale\n"
|
||||
return {"discovered": len(discovered_urls), "new": len(new_rows), "stale": len(stale_urls)}
|
||||
yield f"Listing fetch complete: {len(discovered_urls)} discovered, {len(new_rows)} new,\n"
|
||||
return {"discovered": len(discovered_urls), "new": len(new_rows)}
|
||||
|
||||
|
||||
def process_job_url(job_url: str):
|
||||
|
||||
Reference in New Issue
Block a user